This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 53ad91f88e Single-threaded execution finishes transforms that still
have work af… (#9527)
53ad91f88e is described below
commit 53ad91f88e90b1a03fa9fbbbfe61290242389062
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Tue Nov 17 10:58:13 2026 +0100
Single-threaded execution finishes transforms that still have work af…
(#9527)
* Single-threaded execution finishes transforms that still have work after
their input, fixes #9526
* fix remarks
---
.../pipeline/SingleThreadedPipelineExecutor.java | 57 ++-
...006-kafka-consumer-table-output-subpipeline.hpl | 167 ++++++++
...0006-kafka-consumer-table-output-validation.hpl | 337 ++++++++++++++++
.../kafka/0006-kafka-consumer-table-output.hpl | 103 +++++
.../main-0006-kafka-consumer-table-output.hwf | 206 ++++++++++
.../kafka/metadata/rdbms/h2-kafka.json | 26 ++
.../kafka/prepare-kafka-test-0006-table-output.hpl | 107 +++++
.../0005-csv-filename-from-field-validation.hpl | 282 +++++++++++++
.../0005-csv-filename-from-field.hpl | 157 ++++++++
.../0006-simple-mapping-with-input-child.hpl | 98 +++++
.../0006-simple-mapping-with-input-validation.hpl | 282 +++++++++++++
.../0006-simple-mapping-with-input.hpl | 163 ++++++++
.../0007-group-by-end-of-input-validation.hpl | 312 ++++++++++++++
.../single_threaded/0007-group-by-end-of-input.hpl | 447 +++++++++++++++++++++
.../single_threaded/files/0005-names.csv | 6 +
.../main-0005-csv-filename-from-field.hwf | 138 +++++++
.../main-0006-simple-mapping-with-input.hwf | 138 +++++++
.../main-0007-group-by-end-of-input.hwf | 138 +++++++
.../pipeline/transforms/mapping/SimpleMapping.java | 9 +-
.../transforms/mapping/SimpleMappingTest.java | 22 +
.../transforms/sort/SortRowsBatchesTest.java | 138 +++++++
.../transforms/tableoutput/TableOutput.java | 11 +-
.../transforms/tableoutput/TableOutputTest.java | 30 ++
.../csvinput/CsvInputSingleThreadedTest.java | 198 +++++++++
24 files changed, 3569 insertions(+), 3 deletions(-)
diff --git
a/engine/src/main/java/org/apache/hop/pipeline/SingleThreadedPipelineExecutor.java
b/engine/src/main/java/org/apache/hop/pipeline/SingleThreadedPipelineExecutor.java
index 5257f2017e..b96cbc7e3e 100644
---
a/engine/src/main/java/org/apache/hop/pipeline/SingleThreadedPipelineExecutor.java
+++
b/engine/src/main/java/org/apache/hop/pipeline/SingleThreadedPipelineExecutor.java
@@ -294,6 +294,39 @@ public class SingleThreadedPipelineExecutor {
return true;
}
+ /**
+ * Whether the transform has read all its input: every input row set is
done, has no rows left,
+ * and the transform that writes into it is done. A transform removes a row
set it read to the
+ * end, so no row sets left means the same.
+ *
+ * <p>A row set can be marked done between batches: Sort rows calls
setOutputDone() in every
+ * batchComplete(). In the batch callers (Kafka consumer, Event Hubs
listener) more rows follow in
+ * the next iteration, so only a finished writer means the end of the input.
+ */
+ private boolean isAllInputRead(List<IRowSet> rowSets) {
+ for (IRowSet rowSet : rowSets) {
+ if (!rowSet.isDone() || rowSet.size() > 0 || !isOriginDone(rowSet)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * @return true when the transform that writes into the row set is done. A
row set without such a
+ * transform comes from a RowProducer, which marks it done once
finished() is called.
+ */
+ private boolean isOriginDone(IRowSet rowSet) {
+ for (int i = 0; i < transforms.size(); i++) {
+ TransformMetaDataCombi combi = transforms.get(i);
+ if (combi.transformName.equals(rowSet.getOriginTransformName())
+ && combi.copy == rowSet.getOriginTransformCopy()) {
+ return done[i];
+ }
+ }
+ return true;
+ }
+
/**
* Give all transforms in the pipeline the chance to process all rows on
input...
*
@@ -315,6 +348,10 @@ public class SingleThreadedPipelineExecutor {
// This means that it is impractical to use this pipeline type to
load large files.
//
boolean transformDone = false;
+ // A transform that finishes at the end of its input already wrote
what it held there:
+ // batchComplete() would write it again (Group by, Memory group by,
Unique rows)
+ //
+ boolean finishedAtEndOfInput = false;
// For every input row we call the processRow() method of the
transform.
//
List<IRowSet> infoRowSets = transformInfoRowSets.get(s);
@@ -369,13 +406,31 @@ public class SingleThreadedPipelineExecutor {
return false;
}
}
+
+ // All input arrived and was read, but the transform can still
have work: a reader
+ // going through the file a row named, a mapping, the rows a Table
output still holds.
+ // Like a transform without input, it is called until it says it
is done. Otherwise it
+ // never sees the end of its input, and the single-threaded
engine, which runs one
+ // iteration, disposes it with that work undone.
+ //
+ if (!transformDone &&
isAllInputRead(combi.transform.getInputRowSets())) {
+ while (!transformDone && !pipeline.isStopped()) {
+ transformDone = !combi.transform.processRow();
+ if (combi.transform.getErrors() > 0) {
+ return false;
+ }
+ }
+ }
+ finishedAtEndOfInput = transformDone;
}
// Signal the transform that a batch of rows has passed for this
iteration (sort rows and
// all)
//
try {
- combi.transform.batchComplete();
+ if (!finishedAtEndOfInput) {
+ combi.transform.batchComplete();
+ }
} catch (Exception e) {
combi.transform.setErrors(1);
throw e;
diff --git
a/integration-tests/kafka/0006-kafka-consumer-table-output-subpipeline.hpl
b/integration-tests/kafka/0006-kafka-consumer-table-output-subpipeline.hpl
new file mode 100644
index 0000000000..d5b7559cab
--- /dev/null
+++ b/integration-tests/kafka/0006-kafka-consumer-table-output-subpipeline.hpl
@@ -0,0 +1,167 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0006-kafka-consumer-table-output-subpipeline</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Every batch of 0006-kafka-consumer-table-output goes through
Table output; the rows after it go back to the consumer</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/16 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/16 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Injector</from>
+ <to>Table output</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Table output</from>
+ <to>Out record</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Injector</name>
+ <type>Injector</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>key</name>
+ <type>String</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>message</name>
+ <type>String</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>topic</name>
+ <type>String</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>partition</name>
+ <type>Integer</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>offset</name>
+ <type>Integer</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>timestamp</name>
+ <type>Integer</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Table output</name>
+ <type>TableOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <commit>1000</commit>
+ <connection>h2-kafka</connection>
+ <fields>
+ <field>
+ <column_name>message</column_name>
+ <stream_name>message</stream_name>
+ </field>
+ </fields>
+ <return_field/>
+ <ignore_errors>N</ignore_errors>
+ <partitioning_daily>N</partitioning_daily>
+ <partitioning_enabled>N</partitioning_enabled>
+ <partitioning_field/>
+ <partitioning_monthly>Y</partitioning_monthly>
+ <return_keys>N</return_keys>
+ <schema/>
+ <specify_fields>Y</specify_fields>
+ <table>kafka_table_output</table>
+ <tablename_field/>
+ <tablename_in_field>N</tablename_in_field>
+ <tablename_in_table>Y</tablename_in_table>
+ <truncate>N</truncate>
+ <use_batch>Y</use_batch>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Out record</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>480</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/kafka/0006-kafka-consumer-table-output-validation.hpl
b/integration-tests/kafka/0006-kafka-consumer-table-output-validation.hpl
new file mode 100644
index 0000000000..bafaff636f
--- /dev/null
+++ b/integration-tests/kafka/0006-kafka-consumer-table-output-validation.hpl
@@ -0,0 +1,337 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0006-kafka-consumer-table-output-validation</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Verify 0006-kafka-consumer-table-output: 20 rows came back
after Table output and 20 rows are in the table</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/16 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/16 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>rows after Table output</from>
+ <to>count rows</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>count rows</from>
+ <to>rows: n <> 20</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>rows: n <> 20</from>
+ <to>rows: Abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>rows in the table</from>
+ <to>count table rows</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>count table rows</from>
+ <to>table: n <> 20</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>table: n <> 20</from>
+ <to>table: Abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>rows after Table output</name>
+ <type>RowsFromResult</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>key</name>
+ <type>String</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>message</name>
+ <type>String</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>topic</name>
+ <type>String</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>partition</name>
+ <type>Integer</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>offset</name>
+ <type>Integer</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>timestamp</name>
+ <type>Integer</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>count rows</name>
+ <type>GroupBy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <all_rows>N</all_rows>
+ <ignore_aggregate>N</ignore_aggregate>
+ <field_ignore/>
+ <directory>${java.io.tmpdir}</directory>
+ <prefix>grp</prefix>
+ <add_linenr>N</add_linenr>
+ <linenr_fieldname/>
+ <give_back_row>Y</give_back_row>
+ <group>
+ </group>
+ <fields>
+ <field>
+ <aggregate>n</aggregate>
+ <subject>message</subject>
+ <type>COUNT_ALL</type>
+ <valuefield/>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>rows: n <> 20</name>
+ <type>FilterRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <send_true_to/>
+ <send_false_to/>
+ <compare>
+ <condition>
+ <negated>N</negated>
+ <leftvalue>n</leftvalue>
+ <function><></function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>Integer</type>
+ <text>20</text>
+ <length>-1</length>
+ <precision>0</precision>
+ <isnull>N</isnull>
+ <mask>####0;-####0</mask>
+ </value>
+ </condition>
+ </compare>
+ <attributes/>
+ <GUI>
+ <xloc>480</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>rows: Abort</name>
+ <type>Abort</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <always_log_rows>Y</always_log_rows>
+ <message>0006-kafka-consumer-table-output: not every row came back from
the sub-pipeline after Table output</message>
+ <row_threshold>0</row_threshold>
+ <attributes/>
+ <GUI>
+ <xloc>672</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>rows in the table</name>
+ <type>TableInput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <connection>h2-kafka</connection>
+ <sql>SELECT message FROM kafka_table_output</sql>
+ <limit>0</limit>
+ <lookup/>
+ <execute_each_row>N</execute_each_row>
+ <variables_active>N</variables_active>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>count table rows</name>
+ <type>GroupBy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <all_rows>N</all_rows>
+ <ignore_aggregate>N</ignore_aggregate>
+ <field_ignore/>
+ <directory>${java.io.tmpdir}</directory>
+ <prefix>grp</prefix>
+ <add_linenr>N</add_linenr>
+ <linenr_fieldname/>
+ <give_back_row>Y</give_back_row>
+ <group>
+ </group>
+ <fields>
+ <field>
+ <aggregate>n</aggregate>
+ <subject>message</subject>
+ <type>COUNT_ALL</type>
+ <valuefield/>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>table: n <> 20</name>
+ <type>FilterRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <send_true_to/>
+ <send_false_to/>
+ <compare>
+ <condition>
+ <negated>N</negated>
+ <leftvalue>n</leftvalue>
+ <function><></function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>Integer</type>
+ <text>20</text>
+ <length>-1</length>
+ <precision>0</precision>
+ <isnull>N</isnull>
+ <mask>####0;-####0</mask>
+ </value>
+ </condition>
+ </compare>
+ <attributes/>
+ <GUI>
+ <xloc>480</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>table: Abort</name>
+ <type>Abort</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <always_log_rows>Y</always_log_rows>
+ <message>0006-kafka-consumer-table-output: not every row was written to
kafka_table_output</message>
+ <row_threshold>0</row_threshold>
+ <attributes/>
+ <GUI>
+ <xloc>672</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/kafka/0006-kafka-consumer-table-output.hpl
b/integration-tests/kafka/0006-kafka-consumer-table-output.hpl
new file mode 100644
index 0000000000..74f17116ee
--- /dev/null
+++ b/integration-tests/kafka/0006-kafka-consumer-table-output.hpl
@@ -0,0 +1,103 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0006-kafka-consumer-table-output</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>A Kafka consumer whose sub-pipeline writes through Table
output, 20 messages in batches of 5: every row must come back after Table
output (#9526)</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/16 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/16 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Kafka Consumer</from>
+ <to>Copy rows to result</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Kafka Consumer</name>
+ <type>KafkaConsumer</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <topic>hop-test-table-output</topic>
+ <consumerGroup>hop-it-table-output</consumerGroup>
+
<pipelinePath>${PROJECT_HOME}/0006-kafka-consumer-table-output-subpipeline.hpl</pipelinePath>
+ <subTransform>Out record</subTransform>
+ <batchSize>5</batchSize>
+ <batchDuration>1000</batchDuration>
+ <stopWhenIdle>Y</stopWhenIdle>
+ <maxIdleTimeMs>5000</maxIdleTimeMs>
+ <directBootstrapServers>${BOOTSTRAP_SERVERS}</directBootstrapServers>
+ <AUTO_COMMIT>N</AUTO_COMMIT>
+ <OutputField kafkaName="key" type="String">key</OutputField>
+ <OutputField kafkaName="message" type="String">message</OutputField>
+ <OutputField kafkaName="topic" type="String">topic</OutputField>
+ <OutputField kafkaName="partition" type="Integer">partition</OutputField>
+ <OutputField kafkaName="offset" type="Integer">offset</OutputField>
+ <OutputField kafkaName="timestamp" type="Integer">timestamp</OutputField>
+ <advancedConfig>
+ <option property="auto.offset.reset" value="earliest"/>
+ </advancedConfig>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Copy rows to result</name>
+ <type>RowsToResult</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/kafka/main-0006-kafka-consumer-table-output.hwf
b/integration-tests/kafka/main-0006-kafka-consumer-table-output.hwf
new file mode 100644
index 0000000000..0714c18b5d
--- /dev/null
+++ b/integration-tests/kafka/main-0006-kafka-consumer-table-output.hwf
@@ -0,0 +1,206 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0006-kafka-consumer-table-output</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <created_user>-</created_user>
+ <modified_user>-</modified_user>
+ <created_date>2021/12/30 10:42:13.629</created_date>
+ <modified_date>2021/12/30 10:42:13.629</modified_date>
+ <workflow_version/>
+ <parameters>
+ <parameter>
+ <name>BOOTSTRAP_SERVERS</name>
+ <description>Kafka bootstrap servers (docker IT network
hostname)</description>
+ <default_value>kafka:9092</default_value>
+ </parameter>
+ </parameters>
+ <actions>
+ <action>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <DayOfMonth>1</DayOfMonth>
+ <weekDay>1</weekDay>
+ <minutes>0</minutes>
+ <hour>12</hour>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <xloc>128</xloc>
+ <yloc>144</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>empty table</name>
+ <description/>
+ <type>SQL</type>
+ <attributes/>
+ <connection>h2-kafka</connection>
+ <sendOneStatement>N</sendOneStatement>
+ <sql>CREATE TABLE IF NOT EXISTS kafka_table_output (message
VARCHAR(100));
+DELETE FROM kafka_table_output;</sql>
+ <sqlfromfile>N</sqlfromfile>
+ <useVariableSubstitution>N</useVariableSubstitution>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>224</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>prepare-kafka-test-0006-table-output.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+
<filename>${PROJECT_HOME}/prepare-kafka-test-0006-table-output.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>local</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>352</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0006-kafka-consumer-table-output.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0006-kafka-consumer-table-output.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>local</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>480</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0006-kafka-consumer-table-output-validation.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+
<filename>${PROJECT_HOME}/0006-kafka-consumer-table-output-validation.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>local</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>608</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>empty table</to>
+ <from_nr>0</from_nr>
+ <to_nr>0</to_nr>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>empty table</from>
+ <to>prepare-kafka-test-0006-table-output.hpl</to>
+ <from_nr>0</from_nr>
+ <to_nr>0</to_nr>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>prepare-kafka-test-0006-table-output.hpl</from>
+ <to>0006-kafka-consumer-table-output.hpl</to>
+ <from_nr>0</from_nr>
+ <to_nr>0</to_nr>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>0006-kafka-consumer-table-output.hpl</from>
+ <to>0006-kafka-consumer-table-output-validation.hpl</to>
+ <from_nr>0</from_nr>
+ <to_nr>0</to_nr>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/kafka/metadata/rdbms/h2-kafka.json
b/integration-tests/kafka/metadata/rdbms/h2-kafka.json
new file mode 100644
index 0000000000..434ed7e32a
--- /dev/null
+++ b/integration-tests/kafka/metadata/rdbms/h2-kafka.json
@@ -0,0 +1,26 @@
+{
+ "rdbms": {
+ "H2": {
+ "databaseName": "kafka",
+ "pluginId": "H2",
+ "accessType": 0,
+ "hostname": "",
+ "password": "Encrypted ",
+ "pluginName": "H2",
+ "port": "",
+ "attributes": {
+ "SUPPORTS_TIMESTAMP_DATA_TYPE": "N",
+ "QUOTE_ALL_FIELDS": "N",
+ "SUPPORTS_BOOLEAN_DATA_TYPE": "N",
+ "FORCE_IDENTIFIERS_TO_LOWERCASE": "N",
+ "PRESERVE_RESERVED_WORD_CASE": "Y",
+ "SQL_CONNECT": "",
+ "FORCE_IDENTIFIERS_TO_UPPERCASE": "N",
+ "PREFERRED_SCHEMA_NAME": ""
+ },
+ "manualUrl": "jdbc:h2:mem:kafka_it;DB_CLOSE_DELAY=-1",
+ "username": ""
+ }
+ },
+ "name": "h2-kafka"
+}
diff --git a/integration-tests/kafka/prepare-kafka-test-0006-table-output.hpl
b/integration-tests/kafka/prepare-kafka-test-0006-table-output.hpl
new file mode 100644
index 0000000000..510532f65e
--- /dev/null
+++ b/integration-tests/kafka/prepare-kafka-test-0006-table-output.hpl
@@ -0,0 +1,107 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>prepare-kafka-test-0006-table-output</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Sends 20 messages for
0006-kafka-consumer-table-output</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/16 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/16 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Generate rows</from>
+ <to>Kafka Producer</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Generate rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <length>-1</length>
+ <name>field</name>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <type>String</type>
+ <nullif>Hello Hop!</nullif>
+ </field>
+ </fields>
+ <interval_in_ms>5000</interval_in_ms>
+ <last_time_field>FiveSecondsAgo</last_time_field>
+ <never_ending>N</never_ending>
+ <limit>20</limit>
+ <row_time_field>now</row_time_field>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Kafka Producer</name>
+ <type>KafkaProducerOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <directBootstrapServers>${BOOTSTRAP_SERVERS}</directBootstrapServers>
+ <topic>hop-test-table-output</topic>
+ <clientId/>
+ <keyField/>
+ <messageField>field</messageField>
+ <advancedConfig>
+ <option property="compression.type" value="none"/>
+ </advancedConfig>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/single_threaded/0005-csv-filename-from-field-validation.hpl
b/integration-tests/single_threaded/0005-csv-filename-from-field-validation.hpl
new file mode 100644
index 0000000000..b7d9cdb9cc
--- /dev/null
+++
b/integration-tests/single_threaded/0005-csv-filename-from-field-validation.hpl
@@ -0,0 +1,282 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0005-csv-filename-from-field-validation</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Verify the result rows of 0005-csv-filename-from-field, which
ran on the single threaded engine</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/06 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/06 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Expected</from>
+ <to>Merge rows (diff)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Output of the test</from>
+ <to>Sort output</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Sort output</from>
+ <to>Merge rows (diff)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Merge rows (diff)</from>
+ <to>Not identical</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Not identical</from>
+ <to>Abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Expected</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>id</name>
+ <precision>-1</precision>
+ <type>Integer</type>
+ </field>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>name</name>
+ <precision>-1</precision>
+ <type>String</type>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>1</item>
+ <item>alpha</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>bravo</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item>charlie</item>
+ </line>
+ <line>
+ <item>4</item>
+ <item>delta</item>
+ </line>
+ <line>
+ <item>5</item>
+ <item>echo</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Output of the test</name>
+ <type>RowsFromResult</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>name</name>
+ <type>String</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Sort output</name>
+ <type>SortRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <directory>${java.io.tmpdir}</directory>
+ <sort_prefix>out</sort_prefix>
+ <sort_size>1000000</sort_size>
+ <free_memory/>
+ <unique_rows>N</unique_rows>
+ <compress>N</compress>
+ <fields>
+ <field>
+ <name>id</name>
+ <ascending>Y</ascending>
+ <case_sensitive>N</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ <field>
+ <name>name</name>
+ <ascending>Y</ascending>
+ <case_sensitive>N</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Merge rows (diff)</name>
+ <type>MergeRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <keys>
+ <key>id</key>
+ <key>name</key>
+ </keys>
+ <values>
+ <value>id</value>
+ <value>name</value>
+ </values>
+ <flag_field>flagfield</flag_field>
+ <reference>Expected</reference>
+ <compare>Sort output</compare>
+ <attributes/>
+ <GUI>
+ <xloc>416</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Not identical</name>
+ <type>FilterRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <send_true_to/>
+ <send_false_to/>
+ <compare>
+ <condition>
+ <negated>N</negated>
+ <leftvalue>flagfield</leftvalue>
+ <function><></function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>String</type>
+ <text>identical</text>
+ <length>-1</length>
+ <precision>-1</precision>
+ <isnull>N</isnull>
+ <mask/>
+ </value>
+ </condition>
+ </compare>
+ <attributes/>
+ <GUI>
+ <xloc>576</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Abort</name>
+ <type>Abort</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <always_log_rows>Y</always_log_rows>
+ <message>0005-csv-filename-from-field: not every line of the named file
reached the result on the single threaded engine</message>
+ <row_threshold>0</row_threshold>
+ <attributes/>
+ <GUI>
+ <xloc>736</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/single_threaded/0005-csv-filename-from-field.hpl
b/integration-tests/single_threaded/0005-csv-filename-from-field.hpl
new file mode 100644
index 0000000000..44a442dbdd
--- /dev/null
+++ b/integration-tests/single_threaded/0005-csv-filename-from-field.hpl
@@ -0,0 +1,157 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0005-csv-filename-from-field</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>CSV file input reads the file a row names on the single
threaded engine: every line must reach the result (#9526)</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/16 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/16 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>file name</from>
+ <to>read the named file</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>read the named file</from>
+ <to>Copy rows to result</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>file name</name>
+ <type>GetVariable</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>filename</name>
+ <variable>${PROJECT_HOME}/files/0005-names.csv</variable>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>read the named file</name>
+ <type>CSVInput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <filename/>
+ <filename_field>filename</filename_field>
+ <rownum_field/>
+ <include_filename>N</include_filename>
+ <separator>;</separator>
+ <enclosure>"</enclosure>
+ <header>Y</header>
+ <buffer_size>50000</buffer_size>
+ <lazy_conversion>N</lazy_conversion>
+ <add_filename_result>N</add_filename_result>
+ <parallel>N</parallel>
+ <newline_possible>N</newline_possible>
+ <encoding/>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <format>#</format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>name</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Copy rows to result</name>
+ <type>RowsToResult</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>480</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/single_threaded/0006-simple-mapping-with-input-child.hpl
b/integration-tests/single_threaded/0006-simple-mapping-with-input-child.hpl
new file mode 100644
index 0000000000..4ae9b02a16
--- /dev/null
+++ b/integration-tests/single_threaded/0006-simple-mapping-with-input-child.hpl
@@ -0,0 +1,98 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0006-simple-mapping-with-input-child</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>The mapping of 0006-simple-mapping-with-input: passes its
rows on</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/16 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/16 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Input</from>
+ <to>Output</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Input</name>
+ <type>MappingInput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <length/>
+ <name>id</name>
+ <precision/>
+ <type>Integer</type>
+ </field>
+ <field>
+ <length/>
+ <name>name</name>
+ <precision/>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Output</name>
+ <type>MappingOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/single_threaded/0006-simple-mapping-with-input-validation.hpl
b/integration-tests/single_threaded/0006-simple-mapping-with-input-validation.hpl
new file mode 100644
index 0000000000..7fbbcb1661
--- /dev/null
+++
b/integration-tests/single_threaded/0006-simple-mapping-with-input-validation.hpl
@@ -0,0 +1,282 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0006-simple-mapping-with-input-validation</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Verify the result rows of 0006-simple-mapping-with-input,
which ran on the single threaded engine</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/06 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/06 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Expected</from>
+ <to>Merge rows (diff)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Output of the test</from>
+ <to>Sort output</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Sort output</from>
+ <to>Merge rows (diff)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Merge rows (diff)</from>
+ <to>Not identical</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Not identical</from>
+ <to>Abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Expected</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>id</name>
+ <precision>-1</precision>
+ <type>Integer</type>
+ </field>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>name</name>
+ <precision>-1</precision>
+ <type>String</type>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>1</item>
+ <item>alpha</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>bravo</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item>charlie</item>
+ </line>
+ <line>
+ <item>4</item>
+ <item>delta</item>
+ </line>
+ <line>
+ <item>5</item>
+ <item>echo</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Output of the test</name>
+ <type>RowsFromResult</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>name</name>
+ <type>String</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Sort output</name>
+ <type>SortRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <directory>${java.io.tmpdir}</directory>
+ <sort_prefix>out</sort_prefix>
+ <sort_size>1000000</sort_size>
+ <free_memory/>
+ <unique_rows>N</unique_rows>
+ <compress>N</compress>
+ <fields>
+ <field>
+ <name>id</name>
+ <ascending>Y</ascending>
+ <case_sensitive>N</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ <field>
+ <name>name</name>
+ <ascending>Y</ascending>
+ <case_sensitive>N</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Merge rows (diff)</name>
+ <type>MergeRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <keys>
+ <key>id</key>
+ <key>name</key>
+ </keys>
+ <values>
+ <value>id</value>
+ <value>name</value>
+ </values>
+ <flag_field>flagfield</flag_field>
+ <reference>Expected</reference>
+ <compare>Sort output</compare>
+ <attributes/>
+ <GUI>
+ <xloc>416</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Not identical</name>
+ <type>FilterRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <send_true_to/>
+ <send_false_to/>
+ <compare>
+ <condition>
+ <negated>N</negated>
+ <leftvalue>flagfield</leftvalue>
+ <function><></function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>String</type>
+ <text>identical</text>
+ <length>-1</length>
+ <precision>-1</precision>
+ <isnull>N</isnull>
+ <mask/>
+ </value>
+ </condition>
+ </compare>
+ <attributes/>
+ <GUI>
+ <xloc>576</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Abort</name>
+ <type>Abort</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <always_log_rows>Y</always_log_rows>
+ <message>0006-simple-mapping-with-input: the rows of the Simple mapping on
the single threaded engine are not the expected ones</message>
+ <row_threshold>0</row_threshold>
+ <attributes/>
+ <GUI>
+ <xloc>736</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/single_threaded/0006-simple-mapping-with-input.hpl
b/integration-tests/single_threaded/0006-simple-mapping-with-input.hpl
new file mode 100644
index 0000000000..6fdf3c4bc1
--- /dev/null
+++ b/integration-tests/single_threaded/0006-simple-mapping-with-input.hpl
@@ -0,0 +1,163 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0006-simple-mapping-with-input</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>A Simple mapping with input on the single threaded engine: it
must finish and give every row (#9526)</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/16 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/16 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>rows</from>
+ <to>Simple mapping</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Simple mapping</from>
+ <to>Copy rows to result</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>rows</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>id</name>
+ <precision>-1</precision>
+ <type>Integer</type>
+ </field>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>name</name>
+ <precision>-1</precision>
+ <type>String</type>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>1</item>
+ <item>alpha</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>bravo</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item>charlie</item>
+ </line>
+ <line>
+ <item>4</item>
+ <item>delta</item>
+ </line>
+ <line>
+ <item>5</item>
+ <item>echo</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Simple mapping</name>
+ <type>SimpleMapping</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+
<filename>${PROJECT_HOME}/0006-simple-mapping-with-input-child.hpl</filename>
+ <mappings>
+ <input>
+ <mapping>
+ <main_path>Y</main_path>
+ <rename_on_output>N</rename_on_output>
+ </mapping>
+ </input>
+ <output>
+ <mapping>
+ <main_path>Y</main_path>
+ <rename_on_output>N</rename_on_output>
+ </mapping>
+ </output>
+ <parameters>
+ <inherit_all_vars>Y</inherit_all_vars>
+ </parameters>
+ </mappings>
+ <runConfiguration/>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Copy rows to result</name>
+ <type>RowsToResult</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>480</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/single_threaded/0007-group-by-end-of-input-validation.hpl
b/integration-tests/single_threaded/0007-group-by-end-of-input-validation.hpl
new file mode 100644
index 0000000000..9c473de6b6
--- /dev/null
+++
b/integration-tests/single_threaded/0007-group-by-end-of-input-validation.hpl
@@ -0,0 +1,312 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0007-group-by-end-of-input-validation</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Verify the result rows of 0007-group-by-end-of-input, which
ran on the single threaded engine</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/06 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/06 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Expected</from>
+ <to>Merge rows (diff)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Output of the test</from>
+ <to>Sort output</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Sort output</from>
+ <to>Merge rows (diff)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Merge rows (diff)</from>
+ <to>Not identical</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Not identical</from>
+ <to>Abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Expected</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>kind</name>
+ <precision>-1</precision>
+ <type>String</type>
+ </field>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>grp</name>
+ <precision>-1</precision>
+ <type>String</type>
+ </field>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>total</name>
+ <precision>-1</precision>
+ <type>Integer</type>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>groupby</item>
+ <item>a</item>
+ <item>3</item>
+ </line>
+ <line>
+ <item>groupby</item>
+ <item>b</item>
+ <item>7</item>
+ </line>
+ <line>
+ <item>memgroupby</item>
+ <item>a</item>
+ <item>3</item>
+ </line>
+ <line>
+ <item>memgroupby</item>
+ <item>b</item>
+ <item>7</item>
+ </line>
+ <line>
+ <item>uniquerows</item>
+ <item>a</item>
+ <item>2</item>
+ </line>
+ <line>
+ <item>uniquerows</item>
+ <item>b</item>
+ <item>2</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Output of the test</name>
+ <type>RowsFromResult</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>kind</name>
+ <type>String</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>grp</name>
+ <type>String</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>total</name>
+ <type>Integer</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Sort output</name>
+ <type>SortRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <directory>${java.io.tmpdir}</directory>
+ <sort_prefix>out</sort_prefix>
+ <sort_size>1000000</sort_size>
+ <free_memory/>
+ <unique_rows>N</unique_rows>
+ <compress>N</compress>
+ <fields>
+ <field>
+ <name>kind</name>
+ <ascending>Y</ascending>
+ <case_sensitive>N</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ <field>
+ <name>grp</name>
+ <ascending>Y</ascending>
+ <case_sensitive>N</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ <field>
+ <name>total</name>
+ <ascending>Y</ascending>
+ <case_sensitive>N</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Merge rows (diff)</name>
+ <type>MergeRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <keys>
+ <key>kind</key>
+ <key>grp</key>
+ </keys>
+ <values>
+ <value>total</value>
+ </values>
+ <flag_field>flagfield</flag_field>
+ <reference>Expected</reference>
+ <compare>Sort output</compare>
+ <attributes/>
+ <GUI>
+ <xloc>416</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Not identical</name>
+ <type>FilterRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <send_true_to/>
+ <send_false_to/>
+ <compare>
+ <condition>
+ <negated>N</negated>
+ <leftvalue>flagfield</leftvalue>
+ <function><></function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>String</type>
+ <text>identical</text>
+ <length>-1</length>
+ <precision>-1</precision>
+ <isnull>N</isnull>
+ <mask/>
+ </value>
+ </condition>
+ </compare>
+ <attributes/>
+ <GUI>
+ <xloc>576</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Abort</name>
+ <type>Abort</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <always_log_rows>Y</always_log_rows>
+ <message>0007-group-by-end-of-input: a group was missing, wrong or written
twice on the single threaded engine</message>
+ <row_threshold>0</row_threshold>
+ <attributes/>
+ <GUI>
+ <xloc>736</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/single_threaded/0007-group-by-end-of-input.hpl
b/integration-tests/single_threaded/0007-group-by-end-of-input.hpl
new file mode 100644
index 0000000000..4fda33ad68
--- /dev/null
+++ b/integration-tests/single_threaded/0007-group-by-end-of-input.hpl
@@ -0,0 +1,447 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0007-group-by-end-of-input</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Group by, Memory group by and Unique rows write what they
hold at the end of their input. On the single threaded engine they must write
it once.</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/17 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/17 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Rows</from>
+ <to>Group by</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Group by</from>
+ <to>kind groupby</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>kind groupby</from>
+ <to>layout groupby</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>layout groupby</from>
+ <to>Copy rows to result</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Rows</from>
+ <to>Memory group by</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Memory group by</from>
+ <to>kind memgroupby</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>kind memgroupby</from>
+ <to>layout memgroupby</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>layout memgroupby</from>
+ <to>Copy rows to result</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Rows</from>
+ <to>Unique rows</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Unique rows</from>
+ <to>kind uniquerows</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>kind uniquerows</from>
+ <to>layout uniquerows</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>layout uniquerows</from>
+ <to>Copy rows to result</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Rows</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>N</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>grp</name>
+ <precision>-1</precision>
+ <type>String</type>
+ </field>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>val</name>
+ <precision>-1</precision>
+ <type>Integer</type>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>a</item>
+ <item>1</item>
+ </line>
+ <line>
+ <item>a</item>
+ <item>2</item>
+ </line>
+ <line>
+ <item>b</item>
+ <item>3</item>
+ </line>
+ <line>
+ <item>b</item>
+ <item>4</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>208</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Group by</name>
+ <type>GroupBy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <add_linenr>N</add_linenr>
+ <all_rows>N</all_rows>
+ <directory>${java.io.tmpdir}</directory>
+ <ignore_aggregate>N</ignore_aggregate>
+ <linenr_fieldname/>
+ <prefix>grp</prefix>
+ <give_back_row>N</give_back_row>
+ <group>
+ <field>
+ <name>grp</name>
+ </field>
+ </group>
+ <fields>
+ <field>
+ <aggregate>total</aggregate>
+ <subject>val</subject>
+ <type>SUM</type>
+ <valuefield/>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>kind groupby</name>
+ <type>Constant</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>kind</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif>groupby</nullif>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>464</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>layout groupby</name>
+ <type>SelectValues</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>kind</name>
+ </field>
+ <field>
+ <name>grp</name>
+ </field>
+ <field>
+ <name>total</name>
+ </field>
+ <select_unspecified>N</select_unspecified>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>640</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Memory group by</name>
+ <type>MemoryGroupBy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <give_back_row>N</give_back_row>
+ <group>
+ <field>
+ <name>grp</name>
+ </field>
+ </group>
+ <fields>
+ <field>
+ <aggregate>total</aggregate>
+ <subject>val</subject>
+ <type>SUM</type>
+ <valuefield/>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>208</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>kind memgroupby</name>
+ <type>Constant</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>kind</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif>memgroupby</nullif>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>464</xloc>
+ <yloc>208</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>layout memgroupby</name>
+ <type>SelectValues</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>kind</name>
+ </field>
+ <field>
+ <name>grp</name>
+ </field>
+ <field>
+ <name>total</name>
+ </field>
+ <select_unspecified>N</select_unspecified>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>640</xloc>
+ <yloc>208</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Unique rows</name>
+ <type>Unique</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <count_rows>Y</count_rows>
+ <count_field>total</count_field>
+ <fields>
+ <field>
+ <case_insensitive>N</case_insensitive>
+ <name>grp</name>
+ </field>
+ </fields>
+ <reject_duplicate_row>N</reject_duplicate_row>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>320</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>kind uniquerows</name>
+ <type>Constant</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>kind</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif>uniquerows</nullif>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>464</xloc>
+ <yloc>320</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>layout uniquerows</name>
+ <type>SelectValues</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>kind</name>
+ </field>
+ <field>
+ <name>grp</name>
+ </field>
+ <field>
+ <name>total</name>
+ </field>
+ <select_unspecified>N</select_unspecified>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>640</xloc>
+ <yloc>320</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Copy rows to result</name>
+ <type>RowsToResult</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>832</xloc>
+ <yloc>208</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/single_threaded/files/0005-names.csv
b/integration-tests/single_threaded/files/0005-names.csv
new file mode 100644
index 0000000000..27d7349e8d
--- /dev/null
+++ b/integration-tests/single_threaded/files/0005-names.csv
@@ -0,0 +1,6 @@
+id;name
+1;alpha
+2;bravo
+3;charlie
+4;delta
+5;echo
diff --git
a/integration-tests/single_threaded/main-0005-csv-filename-from-field.hwf
b/integration-tests/single_threaded/main-0005-csv-filename-from-field.hwf
new file mode 100644
index 0000000000..68c4307759
--- /dev/null
+++ b/integration-tests/single_threaded/main-0005-csv-filename-from-field.hwf
@@ -0,0 +1,138 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0005-csv-filename-from-field</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/11/16 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/16 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <start>Y</start>
+ <dummy>N</dummy>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0005-csv-filename-from-field.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0005-csv-filename-from-field.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>single-threaded</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>320</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0005-csv-filename-from-field-validation.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+
<filename>${PROJECT_HOME}/0005-csv-filename-from-field-validation.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>local</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>608</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0005-csv-filename-from-field.hpl</to>
+ <from_nr>0</from_nr>
+ <to_nr>0</to_nr>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>0005-csv-filename-from-field.hpl</from>
+ <to>0005-csv-filename-from-field-validation.hpl</to>
+ <from_nr>0</from_nr>
+ <to_nr>0</to_nr>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/single_threaded/main-0006-simple-mapping-with-input.hwf
b/integration-tests/single_threaded/main-0006-simple-mapping-with-input.hwf
new file mode 100644
index 0000000000..a307b25269
--- /dev/null
+++ b/integration-tests/single_threaded/main-0006-simple-mapping-with-input.hwf
@@ -0,0 +1,138 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0006-simple-mapping-with-input</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/11/16 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/16 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <start>Y</start>
+ <dummy>N</dummy>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0006-simple-mapping-with-input.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0006-simple-mapping-with-input.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>single-threaded</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>320</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0006-simple-mapping-with-input-validation.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+
<filename>${PROJECT_HOME}/0006-simple-mapping-with-input-validation.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>local</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>608</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0006-simple-mapping-with-input.hpl</to>
+ <from_nr>0</from_nr>
+ <to_nr>0</to_nr>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>0006-simple-mapping-with-input.hpl</from>
+ <to>0006-simple-mapping-with-input-validation.hpl</to>
+ <from_nr>0</from_nr>
+ <to_nr>0</to_nr>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/single_threaded/main-0007-group-by-end-of-input.hwf
b/integration-tests/single_threaded/main-0007-group-by-end-of-input.hwf
new file mode 100644
index 0000000000..71d64f6ce5
--- /dev/null
+++ b/integration-tests/single_threaded/main-0007-group-by-end-of-input.hwf
@@ -0,0 +1,138 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0007-group-by-end-of-input</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/11/16 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/16 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <start>Y</start>
+ <dummy>N</dummy>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0007-group-by-end-of-input.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0007-group-by-end-of-input.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>single-threaded</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>320</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0007-group-by-end-of-input-validation.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+
<filename>${PROJECT_HOME}/0007-group-by-end-of-input-validation.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>local</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>608</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0007-group-by-end-of-input.hpl</to>
+ <from_nr>0</from_nr>
+ <to_nr>0</to_nr>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>0007-group-by-end-of-input.hpl</from>
+ <to>0007-group-by-end-of-input-validation.hpl</to>
+ <from_nr>0</from_nr>
+ <to_nr>0</to_nr>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/plugins/transforms/mapping/src/main/java/org/apache/hop/pipeline/transforms/mapping/SimpleMapping.java
b/plugins/transforms/mapping/src/main/java/org/apache/hop/pipeline/transforms/mapping/SimpleMapping.java
index 62c103bc63..f25b7fbc77 100644
---
a/plugins/transforms/mapping/src/main/java/org/apache/hop/pipeline/transforms/mapping/SimpleMapping.java
+++
b/plugins/transforms/mapping/src/main/java/org/apache/hop/pipeline/transforms/mapping/SimpleMapping.java
@@ -127,7 +127,14 @@ public class SimpleMapping extends
BaseTransform<SimpleMappingMeta, SimpleMappin
if (data.rowDataInputMapper != null) {
data.rowDataInputMapper.finished();
}
- data.mappingPipeline.waitUntilFinished();
+ if (data.executor != null) {
+ // A single-threaded mapping only runs when it is driven: nothing
else finishes it
+ while (data.executor.oneIteration() && !isStopped()) {
+ // Keep driving iterations until all transforms in the
sub-pipeline are done
+ }
+ } else {
+ data.mappingPipeline.waitUntilFinished();
+ }
setOutputDone();
return false;
}
diff --git
a/plugins/transforms/mapping/src/test/java/org/apache/hop/pipeline/transforms/mapping/SimpleMappingTest.java
b/plugins/transforms/mapping/src/test/java/org/apache/hop/pipeline/transforms/mapping/SimpleMappingTest.java
index 8c0212283a..601a478048 100644
---
a/plugins/transforms/mapping/src/test/java/org/apache/hop/pipeline/transforms/mapping/SimpleMappingTest.java
+++
b/plugins/transforms/mapping/src/test/java/org/apache/hop/pipeline/transforms/mapping/SimpleMappingTest.java
@@ -336,6 +336,28 @@ class SimpleMappingTest {
verify(transformMockHelper.pipeline, times(1)).startThreads();
}
+ /**
+ * A single-threaded mapping with input only runs while it is driven. After
the last input row
+ * nothing else finishes it, so waiting for it would wait forever.
+ */
+ @Test
+ void testProcessRowEndOfInputDrivesSingleThreadedLoop() throws HopException {
+ when(transformMockHelper.iTransformMeta.isNoInput()).thenReturn(false);
+ when(transformMockHelper.iTransformMeta.isNoOutput()).thenReturn(true);
+ simpleMpData.mappingOutput = null;
+ SingleThreadedPipelineExecutor executor =
mock(SingleThreadedPipelineExecutor.class);
+ when(executor.oneIteration()).thenReturn(true, true, false);
+ simpleMpData.executor = executor;
+
+ smp = newSimpleMapping();
+ smp.addRowSetToInputRowSets(transformMockHelper.getMockInputRowSet(new
Object[] {}));
+
+ assertTrue(smp.processRow(), "the input row goes to the mapping");
+ assertFalse(smp.processRow(), "the end of the input finishes the mapping");
+ verify(executor, times(3)).oneIteration();
+ verify(transformMockHelper.pipeline, never()).waitUntilFinished();
+ }
+
@Test
void initRefusesIncomingHopWhenNoInputThroughTheEngine() {
SimpleMapping mapping = mappingWithIncomingHop();
diff --git
a/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsBatchesTest.java
b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsBatchesTest.java
new file mode 100644
index 0000000000..0023a52613
--- /dev/null
+++
b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsBatchesTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.sort;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.plugins.TransformPluginType;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineHopMeta;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.RowProducer;
+import org.apache.hop.pipeline.SingleThreadedPipelineExecutor;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.ITransformMeta;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
+import org.apache.hop.pipeline.transforms.injector.InjectorMeta;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Kafka consumer and the Event Hubs listener run their sub-pipeline in
batches: rows are put on the
+ * input, then one iteration of the single-threaded executor runs. Sort rows
marks its output done
+ * after every batch, but more batches follow, so the transform after it reads
every batch.
+ */
+class SortRowsBatchesTest {
+
+ private static final int NR_BATCHES = 4;
+ private static final int ROWS_PER_BATCH = 3;
+
+ @BeforeAll
+ static void initHop() throws Exception {
+ HopEnvironment.init();
+ PluginRegistry registry = PluginRegistry.getInstance();
+ for (ITransformMeta meta : List.of(new SortRowsMeta(), new InjectorMeta(),
new DummyMeta())) {
+ if (registry.getPluginId(TransformPluginType.class, meta) == null) {
+ registry.registerPluginClass(
+ meta.getClass().getName(), TransformPluginType.class,
Transform.class);
+ }
+ }
+ }
+
+ @Test
+ void everyBatchReachesTheTransformAfterSortRows() {
+ List<Integer> rowsAfterEachBatch =
+ assertTimeoutPreemptively(Duration.ofSeconds(20),
SortRowsBatchesTest::runBatches);
+
+ assertEquals(List.of(3, 6, 9, 12), rowsAfterEachBatch);
+ }
+
+ private static List<Integer> runBatches() throws Exception {
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ pipelineMeta.setName("sort-rows-batches");
+
+ SortRowsMeta sortRowsMeta = new SortRowsMeta();
+ sortRowsMeta.setDefault();
+ sortRowsMeta.getSortFields().add(new SortRowsField("nr", true, true,
false, 0, false));
+ sortRowsMeta.setDirectory(System.getProperty("java.io.tmpdir"));
+
+ TransformMeta input = addTransform(pipelineMeta, "input", new
InjectorMeta());
+ TransformMeta sort = addTransform(pipelineMeta, "sort", sortRowsMeta);
+ TransformMeta output = addTransform(pipelineMeta, "output", new
DummyMeta());
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(input, sort));
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(sort, output));
+
+ Pipeline pipeline = new LocalPipelineEngine(pipelineMeta);
+ pipeline.setPipelineType(PipelineMeta.PipelineType.SingleThreaded);
+ pipeline.prepareExecution();
+ RowProducer producer = pipeline.addRowProducer("input", 0);
+
+ List<Object[]> rows = new ArrayList<>();
+ pipeline
+ .getTransform("output", 0)
+ .addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) {
+ rows.add(row);
+ }
+ });
+ pipeline.startThreads();
+
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaInteger("nr"));
+
+ SingleThreadedPipelineExecutor executor = new
SingleThreadedPipelineExecutor(pipeline);
+ assertTrue(executor.init());
+ List<Integer> rowsAfterEachBatch = new ArrayList<>();
+ try {
+ long nr = 0;
+ for (int batch = 0; batch < NR_BATCHES; batch++) {
+ for (int r = 0; r < ROWS_PER_BATCH; r++) {
+ producer.putRow(rowMeta, new Object[] {nr++});
+ }
+ executor.oneIteration();
+ rowsAfterEachBatch.add(rows.size());
+ }
+ } finally {
+ executor.dispose();
+ }
+ return rowsAfterEachBatch;
+ }
+
+ private static TransformMeta addTransform(
+ PipelineMeta pipelineMeta, String name, ITransformMeta meta) {
+ String pluginId =
PluginRegistry.getInstance().getPluginId(TransformPluginType.class, meta);
+ TransformMeta transformMeta = new TransformMeta(pluginId, name, meta);
+ pipelineMeta.addTransform(transformMeta);
+ return transformMeta;
+ }
+}
diff --git
a/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutput.java
b/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutput.java
index 6d55d328b6..f2c1448444 100644
---
a/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutput.java
+++
b/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutput.java
@@ -82,6 +82,11 @@ public class TableOutput extends
BaseTransform<TableOutputMeta, TableOutputData>
if (first && meta.isTruncateTable() && !meta.isOnlyWhenHaveRows()) {
truncateTable();
}
+ // The end of the rows: write what is still buffered and say so, as
processRow() must. A
+ // single-threaded run only disposes the transforms once all of them are
done, so the
+ // transforms after this one would otherwise wait for more rows
+ emptyAndCommitBatchBuffers(false);
+ setOutputDone();
return false;
}
@@ -1450,7 +1455,11 @@ public class TableOutput extends
BaseTransform<TableOutputMeta, TableOutputData>
setErrors(1);
stopAll();
} finally {
- setOutputDone();
+ // Only at the end of the run: batchComplete() flushes after every
iteration of a
+ // single-threaded run, and the transforms after this one still read
what follows
+ if (dispose) {
+ setOutputDone();
+ }
if (getErrors() > 0) {
try {
diff --git
a/plugins/transforms/tableoutput/src/test/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutputTest.java
b/plugins/transforms/tableoutput/src/test/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutputTest.java
index 50385e6cbf..f9378dc680 100644
---
a/plugins/transforms/tableoutput/src/test/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutputTest.java
+++
b/plugins/transforms/tableoutput/src/test/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutputTest.java
@@ -32,6 +32,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.sql.Connection;
+import java.util.ArrayList;
import java.util.Map;
import org.apache.hop.core.database.Database;
import org.apache.hop.core.database.DatabaseMeta;
@@ -85,6 +86,9 @@ class TableOutputTest {
tableOutputData.tableName = "sas";
tableOutputData.preparedStatements = mock(Map.class);
tableOutputData.commitCounterMap = mock(Map.class);
+ // The end of the rows flushes what is buffered: nothing here
+ tableOutputData.batchBuffer = new ArrayList<>();
+ tableOutputData.batchBindBuffer = new ArrayList<>();
pipelineMeta = mock(PipelineMeta.class);
doReturn(transformMeta).when(pipelineMeta).findTransform(anyString());
@@ -109,6 +113,32 @@ class TableOutputTest {
doNothing().when(tableOutputSpy).logDetailed(anyString());
}
+ /**
+ * A single-threaded run calls batchComplete() after every iteration. The
transforms after this
+ * one still read the rows that follow, so only the end of the run closes
the output: otherwise a
+ * transform reading an error hop or the output stops after the first quiet
iteration.
+ */
+ @Test
+ void flushingABatchLeavesTheOutputOpenUntilTheEndOfTheRun() throws Exception
{
+ tableOutputSpy.batchComplete();
+ verify(tableOutputSpy, never()).setOutputDone();
+
+ tableOutputSpy.dispose();
+ verify(tableOutputSpy, times(1)).setOutputDone();
+ }
+
+ /**
+ * At the end of its rows the transform writes what it still holds and
closes its output, as
+ * processRow() must: a single-threaded run disposes the transforms only
once all are done.
+ */
+ @Test
+ void theEndOfTheRowsClosesTheOutput() throws Exception {
+ doReturn(null).when(tableOutputSpy).getRow();
+
+ assertFalse(tableOutputSpy.processRow());
+ verify(tableOutputSpy, times(1)).setOutputDone();
+ }
+
@Test
void testWriteToTable() throws Exception {
tableOutputSpy.writeToTable(mock(IRowMeta.class), new Object[] {});
diff --git
a/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputSingleThreadedTest.java
b/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputSingleThreadedTest.java
new file mode 100644
index 0000000000..b1bc8c0a41
--- /dev/null
+++
b/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputSingleThreadedTest.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.csvinput;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.plugins.TransformPluginType;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineHopMeta;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.SingleThreadedPipelineExecutor;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.BaseTransform;
+import org.apache.hop.pipeline.transform.BaseTransformData;
+import org.apache.hop.pipeline.transform.BaseTransformMeta;
+import org.apache.hop.pipeline.transform.ITransform;
+import org.apache.hop.pipeline.transform.ITransformData;
+import org.apache.hop.pipeline.transform.ITransformMeta;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * CSV file input reading the file a row names, run by the {@link
SingleThreadedPipelineExecutor}
+ * for one iteration, as the single-threaded engine runs it. The transform
reads one line per call,
+ * so it needs calls after its input has ended.
+ */
+class CsvInputSingleThreadedTest {
+
+ private static final Duration TIMEOUT = Duration.ofSeconds(20);
+
+ @TempDir Path folder;
+
+ @BeforeAll
+ static void initHop() throws Exception {
+ HopEnvironment.init();
+ PluginRegistry registry = PluginRegistry.getInstance();
+ for (ITransformMeta meta :
+ List.of(new CsvInputMeta(), new DummyMeta(), new
FileNameSourceMeta())) {
+ if (registry.getPluginId(TransformPluginType.class, meta) == null) {
+ registry.registerPluginClass(
+ meta.getClass().getName(), TransformPluginType.class,
Transform.class);
+ }
+ }
+ }
+
+ @Test
+ void theFileARowNamesIsReadToTheEnd() throws Exception {
+ Path file = folder.resolve("orders.csv");
+ StringBuilder text = new StringBuilder("id;name\n");
+ for (int id = 1; id <= 5; id++) {
+ text.append(id).append(";name ").append(id).append('\n');
+ }
+ Files.writeString(file, text.toString());
+
+ List<Object[]> rows = new ArrayList<>();
+ IRowMeta[] rowMeta = new IRowMeta[1];
+ assertTimeoutPreemptively(TIMEOUT, () ->
readSingleThreaded(file.toString(), rows, rowMeta));
+
+ assertEquals(5, rows.size());
+ int id = rowMeta[0].indexOfValue("id");
+ assertEquals(5L, rowMeta[0].getInteger(rows.get(4), id));
+ }
+
+ private static void readSingleThreaded(String filename, List<Object[]> rows,
IRowMeta[] rowMeta)
+ throws Exception {
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ pipelineMeta.setName("csv-input-single-threaded");
+
+ CsvInputMeta csvInputMeta = new CsvInputMeta();
+ csvInputMeta.setFilenameField("filename");
+ csvInputMeta.setDelimiter(";");
+ csvInputMeta.setHeaderPresent(true);
+ CsvInputField idField = new CsvInputField("id");
+ idField.setType(IValueMeta.TYPE_INTEGER);
+ idField.setFormat("#");
+ csvInputMeta.getInputFields().add(idField);
+ csvInputMeta.getInputFields().add(new CsvInputField("name"));
+
+ FileNameSourceMeta fileNameSourceMeta = new FileNameSourceMeta();
+ fileNameSourceMeta.filename = filename;
+ TransformMeta files = addTransform(pipelineMeta, "files",
fileNameSourceMeta);
+ TransformMeta csv = addTransform(pipelineMeta, "csv", csvInputMeta);
+ TransformMeta output = addTransform(pipelineMeta, "output", new
DummyMeta());
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(files, csv));
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(csv, output));
+
+ Pipeline pipeline = new LocalPipelineEngine(pipelineMeta);
+ pipeline.setPipelineType(PipelineMeta.PipelineType.SingleThreaded);
+ pipeline.prepareExecution();
+
+ ITransform csvTransform = pipeline.getTransform("csv", 0);
+ csvTransform.addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta meta, Object[] row) {
+ rowMeta[0] = meta;
+ rows.add(row);
+ }
+ });
+
+ pipeline.startThreads();
+
+ SingleThreadedPipelineExecutor executor = new
SingleThreadedPipelineExecutor(pipeline);
+ assertTrue(executor.init());
+ try {
+ // One iteration, as the single-threaded engine runs
+ executor.oneIteration();
+ } finally {
+ executor.dispose();
+ }
+ assertEquals(0, csvTransform.getErrors());
+ }
+
+ private static TransformMeta addTransform(
+ PipelineMeta pipelineMeta, String name, ITransformMeta meta) {
+ String pluginId =
PluginRegistry.getInstance().getPluginId(TransformPluginType.class, meta);
+ TransformMeta transformMeta = new TransformMeta(pluginId, name, meta);
+ pipelineMeta.addTransform(transformMeta);
+ return transformMeta;
+ }
+
+ /** A source giving one row with a file name, all in its first call, as a
Data grid does. */
+ @Transform(id = "CsvInputSingleThreadedTestFileName", name = "File name
source")
+ public static class FileNameSourceMeta
+ extends BaseTransformMeta<FileNameSource, FileNameSourceData> {
+ String filename;
+
+ @Override
+ public void getFields(
+ IRowMeta rowMeta,
+ String origin,
+ IRowMeta[] info,
+ TransformMeta nextTransform,
+ IVariables variables,
+ IHopMetadataProvider metadataProvider) {
+ rowMeta.addValueMeta(new ValueMetaString("filename"));
+ }
+ }
+
+ public static class FileNameSourceData extends BaseTransformData implements
ITransformData {}
+
+ public static class FileNameSource extends BaseTransform<FileNameSourceMeta,
FileNameSourceData> {
+ public FileNameSource(
+ TransformMeta transformMeta,
+ FileNameSourceMeta meta,
+ FileNameSourceData data,
+ int copyNr,
+ PipelineMeta pipelineMeta,
+ Pipeline pipeline) {
+ super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline);
+ }
+
+ @Override
+ public boolean processRow() throws HopException {
+ IRowMeta rowMeta = new RowMeta();
+ meta.getFields(rowMeta, getTransformName(), null, null, this, null);
+ putRow(rowMeta, new Object[] {meta.filename});
+ setOutputDone();
+ return false;
+ }
+ }
+}