This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 286b0f6d41 Issue #8035 : Add Data set output transform (#8118)
286b0f6d41 is described below
commit 286b0f6d410676671211b5948a27b06fe0083f72
Author: Matt Casters <[email protected]>
AuthorDate: Thu Aug 27 14:08:20 2026 +0200
Issue #8035 : Add Data set output transform (#8118)
Add a Data set output transform so pipelines can write rows into a
unit-test data set (metadata + CSV) from Hop GUI and hop-run, without
hard-coding the data set file path.
The dialog is built from @GuiWidgetElement annotations with a grouped
layout. Rows are streamed through DataSetCsvWriter, which is also used
by Write rows to data set. Recreate builds the data set from incoming
fields; validate checks an existing set before overwrite.
---
.../images/transforms/icons/datasetoutput.svg | 66 +++++++
docs/hop-user-manual/modules/ROOT/nav.adoc | 1 +
.../ROOT/pages/metadata-types/data-set.adoc | 2 +-
.../ROOT/pages/pipeline/pipeline-unit-testing.adoc | 3 +
.../modules/ROOT/pages/pipeline/transforms.adoc | 1 +
.../pages/pipeline/transforms/datasetoutput.adoc | 64 +++++++
.../transforms/0108-data-set-output-generate.hpl | 123 +++++++++++++
.../transforms/0108-data-set-output-verify.hpl | 85 +++++++++
.../datasets/golden-0108-data-set-output.csv | 4 +
.../transforms/main-0108-data-set-output.hwf | 115 +++++++++++++
.../metadata/dataset/0108-data-set-output.json | 23 +++
.../dataset/golden-0108-data-set-output.json | 23 +++
.../0108-data-set-output-verify UNIT.json | 33 ++++
.../main/java/org/apache/hop/testing/DataSet.java | 69 ++++++++
.../org/apache/hop/testing/DataSetCsvUtil.java | 49 +-----
.../org/apache/hop/testing/DataSetCsvWriter.java | 116 +++++++++++++
.../transforms/datasetoutput/DataSetOutput.java | 190 +++++++++++++++++++++
.../datasetoutput/DataSetOutputData.java | 40 +++++
.../datasetoutput/DataSetOutputDialog.java | 120 +++++++++++++
.../datasetoutput/DataSetOutputMeta.java | 177 +++++++++++++++++++
.../testing/xp/WriteToDataSetExtensionPoint.java | 25 +--
.../messages/messages_en_US.properties | 40 +++++
.../apache/hop/testing/DataSetCsvWriterTest.java | 134 +++++++++++++++
.../org/apache/hop/testing/DataSetRowMetaTest.java | 111 ++++++++++++
.../datasetoutput/DataSetOutputMetaTest.java | 51 ++++++
.../test/resources/data-set-output-transform.xml | 40 +++++
26 files changed, 1646 insertions(+), 59 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/datasetoutput.svg
b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/datasetoutput.svg
new file mode 100644
index 0000000000..b88913ebd5
--- /dev/null
+++
b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/datasetoutput.svg
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="utf-8"?>
+<svg version="1.1"
+ xmlns="http://www.w3.org/2000/svg";
+ x="0px" y="0px" width="24px" height="24px" viewBox="-1 -1 24 24"
enable-background="new -1 -1 24 24"
+ xml:space="preserve">
+<rect id="rect1408_54_" x="2" y="12" fill="#5DC2C6" stroke="#5DC2C6"
stroke-linecap="round" stroke-linejoin="round"
+ width="3" height="3"/>
+ <rect id="rect1408_53_" x="7" y="12" fill="#5DC2C6" stroke="#5DC2C6"
stroke-linecap="round" stroke-linejoin="round"
+ width="3" height="3"/>
+ <rect id="rect1408_52_" x="12" y="12" fill="#5DC2C6" stroke="#5DC2C6"
stroke-linecap="round" stroke-linejoin="round"
+ width="3" height="3"/>
+ <rect id="rect1408_51_" x="17" y="12" fill="#5DC2C6" stroke="#5DC2C6"
stroke-linecap="round" stroke-linejoin="round"
+ width="3" height="3"/>
+ <rect id="rect1408_50_" x="2" y="17" fill="#5DC2C6" stroke="#5DC2C6"
stroke-linecap="round" stroke-linejoin="round"
+ width="3" height="3"/>
+ <rect id="rect1408_49_" x="7" y="17" fill="#5DC2C6" stroke="#5DC2C6"
stroke-linecap="round" stroke-linejoin="round"
+ width="3" height="3"/>
+ <rect id="rect1408_48_" x="12" y="17" fill="#5DC2C6" stroke="#5DC2C6"
stroke-linecap="round" stroke-linejoin="round"
+ width="3" height="3"/>
+ <rect id="rect1408_47_" x="17" y="17" fill="#5DC2C6" stroke="#5DC2C6"
stroke-linecap="round" stroke-linejoin="round"
+ width="3" height="3"/>
+ <rect id="rect1408_46_" x="12" y="2" fill="#5DC2C6" stroke="#5DC2C6"
stroke-linecap="round" stroke-linejoin="round"
+ width="3" height="3"/>
+ <rect id="rect1408_45_" x="17" y="2" fill="#5DC2C6" stroke="#5DC2C6"
stroke-linecap="round" stroke-linejoin="round"
+ width="3" height="3"/>
+ <rect id="rect1408_44_" x="7" y="7" fill="#5DC2C6" stroke="#5DC2C6"
stroke-linecap="round" stroke-linejoin="round"
+ width="3" height="3"/>
+ <rect id="rect1408_43_" x="12" y="7" fill="#5DC2C6" stroke="#5DC2C6"
stroke-linecap="round" stroke-linejoin="round"
+ width="3" height="3"/>
+ <rect id="rect1408_42_" x="17" y="7" fill="#5DC2C6" stroke="#5DC2C6"
stroke-linecap="round" stroke-linejoin="round"
+ width="3" height="3"/>
+ <polygon id="polygon6_3_" display="none" fill="#0e3a5a" points="22,18.982
22,18.02 18.98,18.02 18.98,15 18.02,15 18.02,18.02
+ 15,18.02 15,18.982 18.02,18.982 18.02,22 18.98,22 18.98,18.982 "/>
+ <polygon id="polygon6_2_" display="none" fill="#0e3a5a" points="22,18.982
22,18.02 18.98,18.02 18.98,15 18.02,15 18.02,18.02
+ 15,18.02 15,18.982 18.02,18.982 18.02,22 18.98,22 18.98,18.982 "/>
+ <polygon id="input_1_" fill="#0e3a5a" points="1.719,5.43 1.719,7 7,6.997
7,1.771 5.429,1.771 5.429,4.318 1.11,0 0,1.111
+ 4.319,5.43 "/>
+ <rect id="rect1408_41_" x="2" y="12" display="none" fill="#F9C940"
stroke="#F9C940" stroke-linecap="round"
+ stroke-linejoin="round" width="3" height="3"/>
+ <rect id="rect1408_40_" x="7" y="12" display="none" fill="#F9C940"
stroke="#F9C940" stroke-linecap="round"
+ stroke-linejoin="round" width="3" height="3"/>
+ <rect id="rect1408_39_" x="12" y="12" display="none" fill="#F9C940"
stroke="#F9C940" stroke-linecap="round"
+ stroke-linejoin="round" width="3" height="3"/>
+ <rect id="rect1408_38_" x="17" y="12" display="none" fill="#F9C940"
stroke="#F9C940" stroke-linecap="round"
+ stroke-linejoin="round" width="3" height="3"/>
+ <rect id="rect1408_37_" x="2" y="17" display="none" fill="#F9C940"
stroke="#F9C940" stroke-linecap="round"
+ stroke-linejoin="round" width="3" height="3"/>
+ <rect id="rect1408_36_" x="7" y="17" display="none" fill="#F9C940"
stroke="#F9C940" stroke-linecap="round"
+ stroke-linejoin="round" width="3" height="3"/>
+ <rect id="rect1408_35_" x="12" y="17" display="none" fill="#F9C940"
stroke="#F9C940" stroke-linecap="round"
+ stroke-linejoin="round" width="3" height="3"/>
+ <rect id="rect1408_34_" x="17" y="17" display="none" fill="#F9C940"
stroke="#F9C940" stroke-linecap="round"
+ stroke-linejoin="round" width="3" height="3"/>
+ <rect id="rect1408_33_" x="2" y="2" transform="matrix(-0.7072 0.707 -0.707
-0.7072 8.4493 3.5007)" display="none"
+ fill="none" stroke="#F9C940" stroke-linecap="round"
stroke-linejoin="round" width="3" height="3"/>
+ <rect id="rect1408_32_" x="12" y="2" display="none" fill="#F9C940"
stroke="#F9C940" stroke-linecap="round"
+ stroke-linejoin="round" width="3" height="3"/>
+ <rect id="rect1408_31_" x="17" y="2" display="none" fill="#F9C940"
stroke="#F9C940" stroke-linecap="round"
+ stroke-linejoin="round" width="3" height="3"/>
+ <rect id="rect1408_30_" x="7" y="7" display="none" fill="#F9C940"
stroke="#F9C940" stroke-linecap="round"
+ stroke-linejoin="round" width="3" height="3"/>
+ <rect id="rect1408_29_" x="12" y="7" display="none" fill="#F9C940"
stroke="#F9C940" stroke-linecap="round"
+ stroke-linejoin="round" width="3" height="3"/>
+ <rect id="rect1408_20_" x="17" y="7" display="none" fill="#F9C940"
stroke="#F9C940" stroke-linecap="round"
+ stroke-linejoin="round" width="3" height="3"/>
+</svg>
diff --git a/docs/hop-user-manual/modules/ROOT/nav.adoc
b/docs/hop-user-manual/modules/ROOT/nav.adoc
index 7978adfc85..7e3762a580 100644
--- a/docs/hop-user-manual/modules/ROOT/nav.adoc
+++ b/docs/hop-user-manual/modules/ROOT/nav.adoc
@@ -129,6 +129,7 @@ under the License.
*** xref:pipeline/transforms/databaselookup.adoc[Database Lookup]
*** xref:pipeline/transforms/datagrid.adoc[Data Grid]
*** xref:pipeline/transforms/datasetinput.adoc[Data set input]
+*** xref:pipeline/transforms/datasetoutput.adoc[Data set output]
*** xref:pipeline/transforms/dbimpactinput.adoc[Database impact input]
*** xref:pipeline/transforms/ddl.adoc[DDL Generator]
*** xref:pipeline/transforms/validator.adoc[Data Validator]
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/data-set.adoc
b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/data-set.adoc
index 0f8d233afb..602d166976 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/data-set.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/data-set.adoc
@@ -28,7 +28,7 @@ After processing the result of the pipeline is compared to a
data set that has b
== Related Plugins
-None/All
+xref:pipeline/transforms/datasetinput.adoc[Data set input],
xref:pipeline/transforms/datasetoutput.adoc[Data set output]
== Options
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-unit-testing.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-unit-testing.adoc
index 44c7b23f53..4dc86cd1e0 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-unit-testing.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-unit-testing.adoc
@@ -88,6 +88,9 @@ image::pipeline-unit-testing-data-sets.png[Pipeline Unit
Testing - Data Sets,wid
TIP: creating data sets is also possible from the 'New' context menu or
metadata perspective.
+You can also generate or regenerate a data set from a pipeline with the
xref:pipeline/transforms/datasetoutput.adoc[Data set output] transform.
+That is useful in automated or headless (`hop-run`) runs, where the canvas
*Write rows to data set* action is not available.
+
=== Create and add data sets
Consider the following basic pipeline below.
diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc
index 9b61466564..15f47959a2 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc
@@ -80,6 +80,7 @@ Those are explained once in
xref:pipeline/formatting-values.adoc[Formatting numb
* xref:pipeline/transforms/databaselookup.adoc[Database Lookup]
* xref:pipeline/transforms/datagrid.adoc[Data Grid]
* xref:pipeline/transforms/datasetinput.adoc[Data set input]
+* xref:pipeline/transforms/datasetoutput.adoc[Data set output]
* xref:pipeline/transforms/data-stream-input.adoc[Data stream input]
* xref:pipeline/transforms/data-stream-output.adoc[Data stream output]
* xref:pipeline/transforms/dbimpactinput.adoc[Database impact input]
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/datasetoutput.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/datasetoutput.adoc
new file mode 100644
index 0000000000..d4ff1c143e
--- /dev/null
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/datasetoutput.adoc
@@ -0,0 +1,64 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:documentationPath: /pipeline/transforms/
+:language: en_US
+:description: The Data set output transform writes rows to a data set in the
metadata
+
+= image:transforms/icons/datasetoutput.svg[Data set output transform Icon,
role="image-doc-icon"] Data set output
+
+[%noheader,cols="3a,1a", role="table-no-borders" ]
+|===
+|
+== Description
+
+The Data set output transform writes incoming rows to a
xref:metadata-types/data-set.adoc[data set].
+Use it to generate or regenerate unit-test input and golden data sets from a
pipeline, including headless `hop-run` executions, without hard-coding the CSV
path in a Text file output transform.
+
+Rows are passed through unchanged so you can keep using the stream after the
write.
+
+|
+== Supported Engines
+[%noheader,cols="2,1a",frame=none, role="table-supported-engines"]
+!===
+!Hop Engine! image:check_mark.svg[Supported, 24]
+!Single Threaded! image:check_mark.svg[Supported, 24]
+!Native Spark! image:cross.svg[Not Supported, 24]
+!Beam Spark! image:cross.svg[Not Supported, 24]
+!Beam Flink! image:cross.svg[Not Supported, 24]
+!Beam Dataflow! image:cross.svg[Not Supported, 24]
+!===
+|===
+
+== Options
+
+[width="90%", options="header"]
+|===
+|Option|Description
+|Transform name|The name of this transform
+|Data set name|The name of the data set to write to. Select an existing data
set or type a new name when (re)creating.
+|Folder name|Optional folder for the CSV file. Leave empty to use
`'${HOP_DATASETS_FOLDER}'`.
+|CSV filename|Base filename of the data set CSV file. Leave empty to use the
data set name with a `.csv` extension when (re)creating, or the filename stored
on an existing data set.
+|(Re)create data set|When enabled, the transform creates or overwrites the
data set metadata from the incoming row metadata, then writes the CSV
(including a header-only file when the stream is empty).
+|Validate against existing data set|When (re)create is disabled, load the
existing data set and fail if the incoming field count, names (same order,
case-insensitive) or types do not match. Extra input fields are ignored when
validation is off; missing data set fields are written empty.
+|===
+
+== Notes
+
+* The CSV is overwritten on every run.
+* Multiple copies of this transform are not supported.
+* (Re)create takes precedence over validate.
+* The GUI context actions *Create data set* and *Write rows to data set*
remain available for interactive use on the pipeline canvas.
diff --git a/integration-tests/transforms/0108-data-set-output-generate.hpl
b/integration-tests/transforms/0108-data-set-output-generate.hpl
new file mode 100644
index 0000000000..c96a515710
--- /dev/null
+++ b/integration-tests/transforms/0108-data-set-output-generate.hpl
@@ -0,0 +1,123 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <pipeline_version/>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters/>
+ <name>0108-data-set-output-generate</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <created_user>-</created_user>
+ <modified_user>-</modified_user>
+ <created_date>2026/04/26 12:00:00.000</created_date>
+ <modified_date>2026/04/26 12:00:00.000</modified_date>
+ </info>
+ <transform>
+ <type>DataGrid</type>
+ <name>test data</name>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <group/>
+ <name>id</name>
+ <type>Integer</type>
+ <format/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <group/>
+ <name>name</name>
+ <type>String</type>
+ <format/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>1</item>
+ <item>Alice</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>Bob</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item>Carol</item>
+ </line>
+ </data>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>144</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>DataSetOutput</type>
+ <name>write data set</name>
+ <dataSetName>0108-data-set-output</dataSetName>
+ <folderName/>
+ <csvFilename>0108-data-set-output.csv</csvFilename>
+ <recreateDataSet>N</recreateDataSet>
+ <validateDataSet>Y</validateDataSet>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>368</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <order>
+ <hop>
+ <from>test data</from>
+ <to>write data set</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <notepads/>
+ <attributes/>
+ <transform_error_handling/>
+</pipeline>
diff --git a/integration-tests/transforms/0108-data-set-output-verify.hpl
b/integration-tests/transforms/0108-data-set-output-verify.hpl
new file mode 100644
index 0000000000..de58f2ee51
--- /dev/null
+++ b/integration-tests/transforms/0108-data-set-output-verify.hpl
@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0108-data-set-output-verify</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/04/26 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/04/26 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>read data set</from>
+ <to>Validate</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>read data set</name>
+ <type>DataSetInput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <dataSetName>0108-data-set-output</dataSetName>
+ <attributes/>
+ <GUI>
+ <xloc>160</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Validate</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>368</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-0108-data-set-output.csv
b/integration-tests/transforms/datasets/golden-0108-data-set-output.csv
new file mode 100644
index 0000000000..578dd686ed
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-0108-data-set-output.csv
@@ -0,0 +1,4 @@
+id,name
+1,Alice
+2,Bob
+3,Carol
diff --git a/integration-tests/transforms/main-0108-data-set-output.hwf
b/integration-tests/transforms/main-0108-data-set-output.hwf
new file mode 100644
index 0000000000..a196643e25
--- /dev/null
+++ b/integration-tests/transforms/main-0108-data-set-output.hwf
@@ -0,0 +1,115 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0108-data-set-output</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/04/26 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/04/26 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>80</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0108-data-set-output-generate</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0108-data-set-output-generate.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>local</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <xloc>288</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run 0108 tests</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0108-data-set-output-verify UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>544</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0108-data-set-output-generate</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>0108-data-set-output-generate</from>
+ <to>Run 0108 tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/transforms/metadata/dataset/0108-data-set-output.json
b/integration-tests/transforms/metadata/dataset/0108-data-set-output.json
new file mode 100644
index 0000000000..0df7b60e90
--- /dev/null
+++ b/integration-tests/transforms/metadata/dataset/0108-data-set-output.json
@@ -0,0 +1,23 @@
+{
+ "base_filename": "0108-data-set-output.csv",
+ "name": "0108-data-set-output",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "id",
+ "field_format": "0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ }
+ ]
+}
diff --git
a/integration-tests/transforms/metadata/dataset/golden-0108-data-set-output.json
b/integration-tests/transforms/metadata/dataset/golden-0108-data-set-output.json
new file mode 100644
index 0000000000..f5b8147982
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-0108-data-set-output.json
@@ -0,0 +1,23 @@
+{
+ "base_filename": "golden-0108-data-set-output.csv",
+ "name": "golden-0108-data-set-output",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "id",
+ "field_format": "0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ }
+ ]
+}
diff --git
a/integration-tests/transforms/metadata/unit-test/0108-data-set-output-verify
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0108-data-set-output-verify
UNIT.json
new file mode 100644
index 0000000000..36c11a372e
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0108-data-set-output-verify
UNIT.json
@@ -0,0 +1,33 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "name",
+ "data_set_field": "name"
+ }
+ ],
+ "field_order": [
+ "id",
+ "name"
+ ],
+ "data_set_name": "golden-0108-data-set-output",
+ "transform_name": "Validate"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0108-data-set-output-verify UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0108-data-set-output-verify.hpl"
+}
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSet.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSet.java
index 6f87743c56..b3742010c1 100644
--- a/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSet.java
+++ b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSet.java
@@ -98,6 +98,75 @@ public class DataSet extends HopMetadataBase implements
Cloneable, IHopMetadata
return rowMeta;
}
+ /**
+ * Build data set fields from Hop row metadata. Used when (re)creating a
data set from a pipeline
+ * stream.
+ */
+ public static List<DataSetField> createFieldsFromRowMeta(IRowMeta rowMeta) {
+ List<DataSetField> setFields = new ArrayList<>();
+ for (int i = 0; i < rowMeta.size(); i++) {
+ IValueMeta valueMeta = rowMeta.getValueMeta(i);
+ setFields.add(
+ new DataSetField(
+ valueMeta.getName(),
+ valueMeta.getType(),
+ valueMeta.getLength(),
+ valueMeta.getPrecision(),
+ valueMeta.getComments(),
+ valueMeta.getFormatMask()));
+ }
+ return setFields;
+ }
+
+ /**
+ * Compare incoming row metadata with this data set. Field count, names
(case-insensitive, same
+ * order) and types must match because the CSV layout is positional.
+ */
+ public void validateRowMeta(IRowMeta inputRowMeta) throws HopException {
+ IRowMeta setRowMeta = getSetRowMeta();
+ List<String> errors = new ArrayList<>();
+ if (inputRowMeta.size() != setRowMeta.size()) {
+ errors.add(
+ "field count: input has "
+ + inputRowMeta.size()
+ + ", data set '"
+ + Const.NVL(getName(), "")
+ + "' has "
+ + setRowMeta.size());
+ }
+ int n = Math.min(inputRowMeta.size(), setRowMeta.size());
+ for (int i = 0; i < n; i++) {
+ IValueMeta inputMeta = inputRowMeta.getValueMeta(i);
+ IValueMeta setMeta = setRowMeta.getValueMeta(i);
+ if (!inputMeta.getName().equalsIgnoreCase(setMeta.getName())) {
+ errors.add(
+ "field "
+ + (i + 1)
+ + ": name '"
+ + inputMeta.getName()
+ + "' vs '"
+ + setMeta.getName()
+ + "'");
+ }
+ if (inputMeta.getType() != setMeta.getType()) {
+ errors.add(
+ "field '"
+ + inputMeta.getName()
+ + "': type "
+ + inputMeta.getTypeDesc()
+ + " vs "
+ + setMeta.getTypeDesc());
+ }
+ }
+ if (!errors.isEmpty()) {
+ throw new HopException(
+ "Input row metadata does not match data set '"
+ + Const.NVL(getName(), "")
+ + "': "
+ + String.join("; ", errors));
+ }
+ }
+
public DataSetField findFieldWithName(String fieldName) {
for (DataSetField field : fields) {
if (field.getFieldName().equalsIgnoreCase(fieldName)) {
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
index 07cf152302..76f414ef53 100644
---
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
@@ -18,17 +18,12 @@
package org.apache.hop.testing;
import java.io.BufferedInputStream;
-import java.io.BufferedWriter;
-import java.io.IOException;
import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
import java.io.Reader;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
-import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.csv.QuoteMode;
import org.apache.commons.lang3.StringUtils;
@@ -223,49 +218,9 @@ public class DataSetCsvUtil {
public static final void writeDataSetData(
IVariables variables, DataSet dataSet, IRowMeta rowMeta, List<Object[]>
rows)
throws HopException {
-
- String dataSetFilename = dataSet.getActualDataSetFilename(variables);
-
- IRowMeta setRowMeta = rowMeta.clone(); // just making sure
- setValueFormats(setRowMeta);
-
- OutputStream outputStream = null;
- BufferedWriter writer = null;
- CSVPrinter csvPrinter = null;
- try {
-
- FileObject file = HopVfs.getFileObject(dataSetFilename);
- outputStream = HopVfs.getOutputStream(file, false);
- writer = new BufferedWriter(new OutputStreamWriter(outputStream));
- CSVFormat csvFormat = getCsvFormat(rowMeta);
- csvPrinter = new CSVPrinter(writer, csvFormat);
-
+ try (DataSetCsvWriter writer = new DataSetCsvWriter(variables, dataSet,
rowMeta)) {
for (Object[] row : rows) {
- List<String> strings = new ArrayList<>();
- for (int i = 0; i < setRowMeta.size(); i++) {
- IValueMeta valueMeta = setRowMeta.getValueMeta(i);
- String string = valueMeta.getString(row[i]);
- strings.add(string);
- }
- csvPrinter.printRecord(strings);
- }
- csvPrinter.flush();
-
- } catch (Exception e) {
- throw new HopException("Unable to write data set to file '" +
dataSetFilename + "'", e);
- } finally {
- try {
- if (csvPrinter != null) {
- csvPrinter.close();
- }
- if (writer != null) {
- writer.close();
- }
- if (outputStream != null) {
- outputStream.close();
- }
- } catch (IOException e) {
- throw new HopException("Error closing file " + dataSetFilename + " :
", e);
+ writer.writeRow(row);
}
}
}
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvWriter.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvWriter.java
new file mode 100644
index 0000000000..7e95177da8
--- /dev/null
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvWriter.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.testing;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.csv.CSVPrinter;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.vfs.HopVfs;
+
+/**
+ * Streams rows to a data set CSV file. The header is written when the writer
is opened so an empty
+ * input still produces a valid (header-only) data set file.
+ */
+public class DataSetCsvWriter implements AutoCloseable {
+
+ private final IRowMeta setRowMeta;
+ private final String dataSetFilename;
+ private OutputStream outputStream;
+ private BufferedWriter writer;
+ private CSVPrinter csvPrinter;
+ private boolean closed;
+
+ public DataSetCsvWriter(IVariables variables, DataSet dataSet, IRowMeta
rowMeta)
+ throws HopException {
+ this.dataSetFilename = dataSet.getActualDataSetFilename(variables);
+ this.setRowMeta = rowMeta.clone();
+ DataSetCsvUtil.setValueFormats(this.setRowMeta);
+
+ try {
+ FileObject file = HopVfs.getFileObject(dataSetFilename);
+ FileObject parent = file.getParent();
+ if (parent != null && !parent.exists()) {
+ parent.createFolder();
+ }
+ outputStream = HopVfs.getOutputStream(file, false);
+ writer = new BufferedWriter(new OutputStreamWriter(outputStream));
+ csvPrinter = new CSVPrinter(writer,
DataSetCsvUtil.getCsvFormat(this.setRowMeta));
+ } catch (Exception e) {
+ closeQuietly();
+ throw new HopException("Unable to open data set file '" +
dataSetFilename + "'", e);
+ }
+ }
+
+ public void writeRow(Object[] row) throws HopException {
+ if (closed) {
+ throw new HopException("Data set CSV writer is already closed: " +
dataSetFilename);
+ }
+ try {
+ List<String> strings = new ArrayList<>(setRowMeta.size());
+ for (int i = 0; i < setRowMeta.size(); i++) {
+ IValueMeta valueMeta = setRowMeta.getValueMeta(i);
+ strings.add(valueMeta.getString(row[i]));
+ }
+ csvPrinter.printRecord(strings);
+ } catch (Exception e) {
+ throw new HopException("Unable to write a row to data set file '" +
dataSetFilename + "'", e);
+ }
+ }
+
+ @Override
+ public void close() throws HopException {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ try {
+ if (csvPrinter != null) {
+ csvPrinter.flush();
+ csvPrinter.close();
+ csvPrinter = null;
+ }
+ if (writer != null) {
+ writer.close();
+ writer = null;
+ }
+ if (outputStream != null) {
+ outputStream.close();
+ outputStream = null;
+ }
+ } catch (IOException e) {
+ throw new HopException("Error closing data set file '" + dataSetFilename
+ "'", e);
+ }
+ }
+
+ private void closeQuietly() {
+ try {
+ close();
+ } catch (HopException e) {
+ // Best effort while handling an open failure
+ }
+ }
+}
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetoutput/DataSetOutput.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetoutput/DataSetOutput.java
new file mode 100644
index 0000000000..354c3d7cf8
--- /dev/null
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetoutput/DataSetOutput.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.testing.transforms.datasetoutput;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowDataUtil;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.IHopMetadataSerializer;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.BaseTransform;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.testing.DataSet;
+import org.apache.hop.testing.DataSetCsvWriter;
+
+public class DataSetOutput extends BaseTransform<DataSetOutputMeta,
DataSetOutputData> {
+ private static final Class<?> PKG = DataSetOutputMeta.class;
+
+ public DataSetOutput(
+ TransformMeta transformMeta,
+ DataSetOutputMeta meta,
+ DataSetOutputData data,
+ int copyNr,
+ PipelineMeta pipelineMeta,
+ Pipeline pipeline) {
+ super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline);
+ }
+
+ @Override
+ public boolean init() {
+ if (getTransformMeta().getCopies(this) > 1) {
+ logError(BaseMessages.getString(PKG, "DataSetOutput.Log.OnlyOneCopy"));
+ return false;
+ }
+ if (StringUtils.isEmpty(meta.getDataSetName())) {
+ logError(BaseMessages.getString(PKG,
"DataSetOutput.Log.DataSetNameMissing"));
+ return false;
+ }
+ data.realDataSetName = resolve(meta.getDataSetName());
+ if (StringUtils.isEmpty(data.realDataSetName)) {
+ logError(BaseMessages.getString(PKG,
"DataSetOutput.Log.DataSetNameMissing"));
+ return false;
+ }
+ data.realFolderName = Const.NVL(resolve(Const.NVL(meta.getFolderName(),
"")), "");
+ data.realCsvFilename = Const.NVL(resolve(Const.NVL(meta.getCsvFilename(),
"")), "");
+ return super.init();
+ }
+
+ @Override
+ public boolean processRow() throws HopException {
+ Object[] row = getRow();
+
+ if (first) {
+ first = false;
+ prepareDataSet(getInputRowMeta());
+ }
+
+ if (row == null) {
+ closeWriter();
+ setOutputDone();
+ return false;
+ }
+
+ writeMappedRow(row);
+ putRow(getInputRowMeta(), row);
+ return true;
+ }
+
+ private void prepareDataSet(IRowMeta inputRowMeta) throws HopException {
+ IRowMeta rowMeta = inputRowMeta;
+ if (rowMeta == null) {
+ rowMeta = getPipelineMeta().getPrevTransformFields(this,
getTransformMeta());
+ }
+ if (rowMeta == null || rowMeta.isEmpty()) {
+ throw new HopException(BaseMessages.getString(PKG,
"DataSetOutput.Log.NoInputRowMeta"));
+ }
+
+ IHopMetadataSerializer<DataSet> serializer =
metadataProvider.getSerializer(DataSet.class);
+
+ if (meta.isRecreateDataSet()) {
+ data.dataSet = serializer.load(data.realDataSetName);
+ if (data.dataSet == null) {
+ data.dataSet = new DataSet();
+ data.dataSet.setName(data.realDataSetName);
+ }
+ if (StringUtils.isNotEmpty(data.realFolderName)) {
+ data.dataSet.setFolderName(data.realFolderName);
+ }
+ if (StringUtils.isNotEmpty(data.realCsvFilename)) {
+ data.dataSet.setBaseFilename(data.realCsvFilename);
+ } else if (StringUtils.isEmpty(data.dataSet.getBaseFilename())) {
+ data.dataSet.setBaseFilename(data.realDataSetName + ".csv");
+ }
+ data.dataSet.setFields(DataSet.createFieldsFromRowMeta(rowMeta));
+ serializer.save(data.dataSet);
+ data.setRowMeta = data.dataSet.getSetRowMeta();
+ data.fieldIndexes = identityIndexes(rowMeta.size());
+ } else {
+ DataSet existing = serializer.load(data.realDataSetName);
+ if (existing == null) {
+ throw new HopException(
+ BaseMessages.getString(PKG, "DataSetOutput.Log.DataSetNotFound",
data.realDataSetName));
+ }
+ if (meta.isValidateDataSet()) {
+ existing.validateRowMeta(rowMeta);
+ }
+ String folder =
+ StringUtils.isNotEmpty(data.realFolderName)
+ ? data.realFolderName
+ : existing.getFolderName();
+ String filename =
+ StringUtils.isNotEmpty(data.realCsvFilename)
+ ? data.realCsvFilename
+ : existing.getBaseFilename();
+ data.dataSet =
+ new DataSet(
+ existing.getName(),
+ existing.getDescription(),
+ folder,
+ filename,
+ existing.getFields());
+ data.setRowMeta = existing.getSetRowMeta();
+ data.fieldIndexes = mapInputToDataSet(rowMeta, data.setRowMeta);
+ }
+
+ data.writer = new DataSetCsvWriter(this, data.dataSet, data.setRowMeta);
+ }
+
+ private int[] identityIndexes(int size) {
+ int[] indexes = new int[size];
+ for (int i = 0; i < size; i++) {
+ indexes[i] = i;
+ }
+ return indexes;
+ }
+
+ private int[] mapInputToDataSet(IRowMeta inputRowMeta, IRowMeta setRowMeta) {
+ int[] indexes = new int[setRowMeta.size()];
+ for (int i = 0; i < setRowMeta.size(); i++) {
+ indexes[i] =
inputRowMeta.indexOfValue(setRowMeta.getValueMeta(i).getName());
+ }
+ return indexes;
+ }
+
+ private void writeMappedRow(Object[] row) throws HopException {
+ Object[] setRow = RowDataUtil.allocateRowData(data.setRowMeta.size());
+ for (int i = 0; i < data.fieldIndexes.length; i++) {
+ int index = data.fieldIndexes[i];
+ if (index >= 0 && index < row.length) {
+ setRow[i] = row[index];
+ }
+ }
+ data.writer.writeRow(setRow);
+ }
+
+ private void closeWriter() throws HopException {
+ if (data.writer != null) {
+ data.writer.close();
+ data.writer = null;
+ }
+ }
+
+ @Override
+ public void dispose() {
+ try {
+ closeWriter();
+ } catch (HopException e) {
+ logError("Error closing data set CSV file", e);
+ }
+ super.dispose();
+ }
+}
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetoutput/DataSetOutputData.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetoutput/DataSetOutputData.java
new file mode 100644
index 0000000000..409f5620f2
--- /dev/null
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetoutput/DataSetOutputData.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.testing.transforms.datasetoutput;
+
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.pipeline.transform.BaseTransformData;
+import org.apache.hop.pipeline.transform.ITransformData;
+import org.apache.hop.testing.DataSet;
+import org.apache.hop.testing.DataSetCsvWriter;
+
+@SuppressWarnings("java:S1104")
+public class DataSetOutputData extends BaseTransformData implements
ITransformData {
+
+ public String realDataSetName;
+ public String realFolderName;
+ public String realCsvFilename;
+ public DataSet dataSet;
+ public IRowMeta setRowMeta;
+ public int[] fieldIndexes;
+ public DataSetCsvWriter writer;
+
+ public DataSetOutputData() {
+ // Do nothing
+ }
+}
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetoutput/DataSetOutputDialog.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetoutput/DataSetOutputDialog.java
new file mode 100644
index 0000000000..d0de0e3c0c
--- /dev/null
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetoutput/DataSetOutputDialog.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.testing.transforms.datasetoutput;
+
+import org.apache.hop.core.util.Utils;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.ui.core.dialog.BaseDialog;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgets;
+import org.apache.hop.ui.core.gui.IGuiPluginCompositeWidgetsListener;
+import org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
+import org.eclipse.swt.widgets.Button;
+import org.eclipse.swt.widgets.Control;
+import org.eclipse.swt.widgets.Shell;
+
+public class DataSetOutputDialog extends BaseTransformDialog {
+ private static final Class<?> PKG = DataSetOutputMeta.class;
+
+ private final DataSetOutputMeta input;
+ private GuiCompositeWidgets widgets;
+
+ public DataSetOutputDialog(
+ Shell parent,
+ IVariables variables,
+ DataSetOutputMeta transformMeta,
+ PipelineMeta pipelineMeta) {
+ super(parent, variables, transformMeta, pipelineMeta);
+ input = transformMeta;
+ }
+
+ @Override
+ public String open() {
+ createShell(BaseMessages.getString(PKG,
"DataSetOutputDialog.Shell.Title"));
+
+ buildButtonBar().ok(e -> ok()).cancel(e -> cancel()).build();
+
+ changed = input.hasChanged();
+
+ widgets =
+ GuiCompositeWidgets.addScrolledComposite(
+ shell,
+ variables,
+ wTransformName,
+ wOk,
+ DataSetOutputMeta.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ input);
+ widgets.setCompositeWidgetsListener(
+ new IGuiPluginCompositeWidgetsListener() {
+ @Override
+ public void widgetsCreated(GuiCompositeWidgets compositeWidgets) {
+ // No extra layout after creation
+ }
+
+ @Override
+ public void widgetsPopulated(GuiCompositeWidgets compositeWidgets) {
+ updateValidateEnabled();
+ }
+
+ @Override
+ public void widgetModified(
+ GuiCompositeWidgets compositeWidgets, Control changedWidget,
String widgetId) {
+ input.setChanged();
+ if (DataSetOutputMeta.WIDGET_RECREATE.equals(widgetId)) {
+ updateValidateEnabled();
+ }
+ }
+
+ @Override
+ public void persistContents(GuiCompositeWidgets compositeWidgets) {
+ // Contents are persisted when the dialog is closed with OK
+ }
+ });
+ updateValidateEnabled();
+
+ focusTransformName();
+ BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel());
+ return transformName;
+ }
+
+ private void updateValidateEnabled() {
+ Control recreate =
widgets.getWidgetsMap().get(DataSetOutputMeta.WIDGET_RECREATE);
+ Control validate =
widgets.getWidgetsMap().get(DataSetOutputMeta.WIDGET_VALIDATE);
+ if (recreate instanceof Button recreateButton && validate instanceof
Button validateButton) {
+ validateButton.setEnabled(!recreateButton.getSelection());
+ }
+ }
+
+ private void cancel() {
+ transformName = null;
+ input.setChanged(changed);
+ dispose();
+ }
+
+ private void ok() {
+ if (Utils.isEmpty(wTransformName.getText())) {
+ return;
+ }
+
+ widgets.getWidgetsContents(input,
DataSetOutputMeta.GUI_PLUGIN_ELEMENT_PARENT_ID);
+ transformName = wTransformName.getText();
+ input.setChanged();
+ dispose();
+ }
+}
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetoutput/DataSetOutputMeta.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetoutput/DataSetOutputMeta.java
new file mode 100644
index 0000000000..f286660a47
--- /dev/null
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetoutput/DataSetOutputMeta.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.testing.transforms.datasetoutput;
+
+import static org.apache.hop.core.ICheckResult.TYPE_RESULT_ERROR;
+import static org.apache.hop.core.ICheckResult.TYPE_RESULT_OK;
+
+import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.CheckResult;
+import org.apache.hop.core.ICheckResult;
+import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.gui.plugin.GuiWidgetGroupType;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.BaseTransformMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.testing.DataSet;
+
+@Getter
+@Setter
+@Transform(
+ id = "DataSetOutput",
+ description = "Write rows to a data set defined in the metadata",
+ name = "Data set output",
+ image = "write-to-dataset.svg",
+ categoryDescription =
"i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Output",
+ keywords = "i18n::DataSetOutputMeta.keyword",
+ documentationUrl = "/pipeline/transforms/datasetoutput.html")
+@GuiPlugin
+public class DataSetOutputMeta extends BaseTransformMeta<DataSetOutput,
DataSetOutputData> {
+ private static final Class<?> PKG = DataSetOutputMeta.class;
+ public static final String GUI_PLUGIN_ELEMENT_PARENT_ID =
"DATA_SET_OUTPUT_DIALOG_OPTIONS";
+ public static final String WIDGET_DATA_SET_NAME = "dataSetName";
+ public static final String WIDGET_FOLDER_NAME = "folderName";
+ public static final String WIDGET_CSV_FILENAME = "csvFilename";
+ public static final String WIDGET_RECREATE = "recreateDataSet";
+ public static final String WIDGET_VALIDATE = "validateDataSet";
+
+ @GuiWidgetElement(
+ id = WIDGET_DATA_SET_NAME,
+ order = "0100",
+ type = GuiElementType.METADATA,
+ metadata = DataSet.class,
+ label = "i18n::DataSetOutputMeta.DataSetName.Label",
+ toolTip = "i18n::DataSetOutputMeta.DataSetName.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = "Data Set")
+ @HopMetadataProperty(hopMetadataPropertyType =
HopMetadataPropertyType.PIPELINE_DATA_SET)
+ private String dataSetName;
+
+ @GuiWidgetElement(
+ id = WIDGET_FOLDER_NAME,
+ order = "0200",
+ type = GuiElementType.FOLDER,
+ label = "i18n::DataSetOutputMeta.FolderName.Label",
+ toolTip = "i18n::DataSetOutputMeta.FolderName.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = "Data Set")
+ @HopMetadataProperty
+ private String folderName;
+
+ @GuiWidgetElement(
+ id = WIDGET_CSV_FILENAME,
+ order = "0300",
+ type = GuiElementType.TEXT,
+ label = "i18n::DataSetOutputMeta.CsvFilename.Label",
+ toolTip = "i18n::DataSetOutputMeta.CsvFilename.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = "Data Set")
+ @HopMetadataProperty
+ private String csvFilename;
+
+ @GuiWidgetElement(
+ id = WIDGET_RECREATE,
+ order = "0400",
+ type = GuiElementType.CHECKBOX,
+ label = "i18n::DataSetOutputMeta.Recreate.Label",
+ toolTip = "i18n::DataSetOutputMeta.Recreate.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = "Data Set")
+ @HopMetadataProperty
+ private boolean recreateDataSet;
+
+ @GuiWidgetElement(
+ id = WIDGET_VALIDATE,
+ order = "0500",
+ type = GuiElementType.CHECKBOX,
+ label = "i18n::DataSetOutputMeta.Validate.Label",
+ toolTip = "i18n::DataSetOutputMeta.Validate.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = "Data Set")
+ @HopMetadataProperty
+ private boolean validateDataSet;
+
+ public DataSetOutputMeta() {
+ super();
+ this.recreateDataSet = true;
+ }
+
+ @Override
+ public void setDefault() {
+ recreateDataSet = true;
+ validateDataSet = false;
+ }
+
+ @Override
+ public void check(
+ List<ICheckResult> remarks,
+ PipelineMeta pipelineMeta,
+ TransformMeta transformMeta,
+ IRowMeta prev,
+ String[] input,
+ String[] output,
+ IRowMeta info,
+ IVariables variables,
+ IHopMetadataProvider metadataProvider) {
+ if (StringUtils.isEmpty(dataSetName)) {
+ remarks.add(
+ new CheckResult(
+ TYPE_RESULT_ERROR,
+ BaseMessages.getString(PKG,
"DataSetOutputMeta.CheckResult.DataSetNameMissing"),
+ transformMeta));
+ } else {
+ remarks.add(
+ new CheckResult(
+ TYPE_RESULT_OK,
+ BaseMessages.getString(PKG,
"DataSetOutputMeta.CheckResult.DataSetNameOK"),
+ transformMeta));
+ }
+
+ if (input.length > 0) {
+ remarks.add(
+ new CheckResult(
+ TYPE_RESULT_OK,
+ BaseMessages.getString(
+ PKG,
"DataSetOutputMeta.CheckResult.ReceivingInfoFromOtherTransforms"),
+ transformMeta));
+ } else {
+ remarks.add(
+ new CheckResult(
+ TYPE_RESULT_ERROR,
+ BaseMessages.getString(PKG,
"DataSetOutputMeta.CheckResult.NoInputReceived"),
+ transformMeta));
+ }
+ }
+}
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/WriteToDataSetExtensionPoint.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/WriteToDataSetExtensionPoint.java
index d4743e8a2a..7b7e67fa9b 100644
---
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/WriteToDataSetExtensionPoint.java
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/WriteToDataSetExtensionPoint.java
@@ -17,7 +17,6 @@
package org.apache.hop.testing.xp;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -37,7 +36,7 @@ import org.apache.hop.pipeline.engine.IPipelineEngine;
import org.apache.hop.pipeline.transform.RowAdapter;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.testing.DataSet;
-import org.apache.hop.testing.DataSetCsvUtil;
+import org.apache.hop.testing.DataSetCsvWriter;
import org.apache.hop.testing.util.DataSetConst;
@ExtensionPoint(
@@ -122,8 +121,14 @@ public class WriteToDataSetExtensionPoint
final IRowMeta setRowMeta = dataSet.getSetRowMeta();
IEngineComponent component =
pipeline.findComponent(transformMeta.getName(), 0);
+ if (component == null) {
+ throw new HopException(
+ "Could not find pipeline component '"
+ + transformMeta.getName()
+ + "' to write to a data set");
+ }
- final List<Object[]> transformsForDbRows = new ArrayList<>();
+ final DataSetCsvWriter writer = new DataSetCsvWriter(pipeline, dataSet,
setRowMeta);
component.addRowListener(
new RowAdapter() {
@@ -133,16 +138,14 @@ public class WriteToDataSetExtensionPoint
for (SourceToTargetMapping mapping : mappings) {
transformForDbRow[mapping.getTargetPosition()] =
row[mapping.getSourcePosition()];
}
- transformsForDbRows.add(transformForDbRow);
+ try {
+ writer.writeRow(transformForDbRow);
+ } catch (HopException e) {
+ throw new HopTransformException(e);
+ }
}
});
- // At the end of the pipeline, write it...
- //
- pipeline.addExecutionFinishedListener(
- engine ->
- // Write it
- //
- DataSetCsvUtil.writeDataSetData(pipeline, dataSet, setRowMeta,
transformsForDbRows));
+ pipeline.addExecutionFinishedListener(engine -> writer.close());
}
}
diff --git
a/plugins/misc/testing/src/main/resources/org/apache/hop/testing/transforms/datasetoutput/messages/messages_en_US.properties
b/plugins/misc/testing/src/main/resources/org/apache/hop/testing/transforms/datasetoutput/messages/messages_en_US.properties
new file mode 100644
index 0000000000..61a73efc92
--- /dev/null
+++
b/plugins/misc/testing/src/main/resources/org/apache/hop/testing/transforms/datasetoutput/messages/messages_en_US.properties
@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+DataSetOutputMeta.keyword=dataset,unittest,golden,fixture,testing,output
+DataSetOutputMeta.CheckResult.DataSetNameMissing=A data set name is not
specified
+DataSetOutputMeta.CheckResult.DataSetNameOK=A data set name is specified
+DataSetOutputMeta.CheckResult.ReceivingInfoFromOtherTransforms=Transform is
receiving input from other transforms
+DataSetOutputMeta.CheckResult.NoInputReceived=No input received from other
transforms
+
+DataSetOutputDialog.Shell.Title=Data set output
+DataSetOutputMeta.DataSetName.Label=Data set name
+DataSetOutputMeta.DataSetName.Tooltip=The name of the data set to write to.
You can select an existing data set or type a new name when (re)creating.
+DataSetOutputMeta.FolderName.Label=Folder name
+DataSetOutputMeta.FolderName.Tooltip=Optional folder for the CSV file. Leave
empty to use '${HOP_DATASETS_FOLDER}'.
+DataSetOutputMeta.CsvFilename.Label=CSV filename
+DataSetOutputMeta.CsvFilename.Tooltip=Base filename of the data set CSV file
(for example customers.csv). Leave empty to use the data set name with a .csv
extension.
+DataSetOutputMeta.Recreate.Label=(Re)create data set
+DataSetOutputMeta.Recreate.Tooltip=Create or overwrite the data set metadata
using the incoming row metadata, then write the CSV file.
+DataSetOutputMeta.Validate.Label=Validate against existing data set
+DataSetOutputMeta.Validate.Tooltip=When not (re)creating, load the existing
data set and fail if the incoming field names, order or types do not match.
+
+DataSetOutput.Log.OnlyOneCopy=The Data set output transform does not support
multiple copies
+DataSetOutput.Log.DataSetNameMissing=Please specify the name of the data set
to write to
+DataSetOutput.Log.NoInputRowMeta=No input row metadata is available for Data
set output
+DataSetOutput.Log.DataSetNotFound=Data set ''{0}'' could not be found in the
metadata
diff --git
a/plugins/misc/testing/src/test/java/org/apache/hop/testing/DataSetCsvWriterTest.java
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/DataSetCsvWriterTest.java
new file mode 100644
index 0000000000..da5431180b
--- /dev/null
+++
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/DataSetCsvWriterTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.testing;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.variables.Variables;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+class DataSetCsvWriterTest {
+
+ @TempDir Path tempDir;
+
+ @BeforeAll
+ static void initHop() throws HopException {
+ HopEnvironment.init();
+ }
+
+ @Test
+ void writeAndReadRoundTrip() throws Exception {
+ Variables variables = new Variables();
+ variables.setVariable(DataSet.VARIABLE_HOP_DATASETS_FOLDER,
tempDir.toString());
+
+ DataSet dataSet = new DataSet();
+ dataSet.setName("roundtrip");
+ dataSet.setBaseFilename("roundtrip.csv");
+ dataSet.setFields(
+ List.of(
+ new DataSetField("id", IValueMeta.TYPE_INTEGER, -1, 0, "", "0"),
+ new DataSetField("name", IValueMeta.TYPE_STRING, -1, -1, "", "")));
+
+ IRowMeta rowMeta = dataSet.getSetRowMeta();
+ try (DataSetCsvWriter writer = new DataSetCsvWriter(variables, dataSet,
rowMeta)) {
+ writer.writeRow(new Object[] {1L, "Alice"});
+ writer.writeRow(new Object[] {2L, "Bob"});
+ }
+
+ List<Object[]> rows = DataSetCsvUtil.getAllRows(variables, dataSet);
+ assertEquals(2, rows.size());
+ assertEquals(1L, rows.get(0)[0]);
+ assertEquals("Alice", rows.get(0)[1]);
+ assertEquals(2L, rows.get(1)[0]);
+ assertEquals("Bob", rows.get(1)[1]);
+ }
+
+ @Test
+ void emptyStreamWritesHeaderOnly() throws Exception {
+ Variables variables = new Variables();
+ variables.setVariable(DataSet.VARIABLE_HOP_DATASETS_FOLDER,
tempDir.toString());
+
+ DataSet dataSet = new DataSet();
+ dataSet.setName("empty");
+ dataSet.setBaseFilename("empty.csv");
+ dataSet.setFields(List.of(new DataSetField("id", IValueMeta.TYPE_INTEGER,
-1, 0, "", "0")));
+
+ IRowMeta rowMeta = dataSet.getSetRowMeta();
+ try (DataSetCsvWriter writer = new DataSetCsvWriter(variables, dataSet,
rowMeta)) {
+ // no rows
+ }
+
+ Path csv = tempDir.resolve("empty.csv");
+ assertTrue(Files.exists(csv));
+ String content = Files.readString(csv);
+ assertTrue(content.contains("id"));
+ assertEquals(0, DataSetCsvUtil.getAllRows(variables, dataSet).size());
+ }
+
+ @Test
+ void writeDataSetDataUsesStreamingWriter() throws Exception {
+ Variables variables = new Variables();
+ variables.setVariable(DataSet.VARIABLE_HOP_DATASETS_FOLDER,
tempDir.toString());
+
+ DataSet dataSet = new DataSet();
+ dataSet.setName("batch");
+ dataSet.setBaseFilename("batch.csv");
+ dataSet.setFields(List.of(new DataSetField("name", IValueMeta.TYPE_STRING,
-1, -1, "", "")));
+
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("name"));
+ DataSetCsvUtil.writeDataSetData(
+ variables, dataSet, rowMeta, List.of(new Object[] {"one"}, new
Object[] {"two"}));
+
+ List<Object[]> rows = DataSetCsvUtil.getAllRows(variables, dataSet);
+ assertEquals(2, rows.size());
+ assertEquals("one", rows.get(0)[0]);
+ assertEquals("two", rows.get(1)[0]);
+ }
+
+ @Test
+ void createsMissingParentFolder() throws Exception {
+ Variables variables = new Variables();
+ Path nested = tempDir.resolve("nested").resolve("sets");
+ variables.setVariable(DataSet.VARIABLE_HOP_DATASETS_FOLDER,
nested.toString());
+
+ DataSet dataSet = new DataSet();
+ dataSet.setName("nested");
+ dataSet.setBaseFilename("nested.csv");
+ dataSet.setFields(List.of(new DataSetField("id", IValueMeta.TYPE_INTEGER,
-1, 0, "", "0")));
+
+ try (DataSetCsvWriter writer =
+ new DataSetCsvWriter(variables, dataSet, dataSet.getSetRowMeta())) {
+ writer.writeRow(new Object[] {7L});
+ }
+
+ assertTrue(Files.exists(nested.resolve("nested.csv")));
+ }
+}
diff --git
a/plugins/misc/testing/src/test/java/org/apache/hop/testing/DataSetRowMetaTest.java
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/DataSetRowMetaTest.java
new file mode 100644
index 0000000000..a9522d9ecf
--- /dev/null
+++
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/DataSetRowMetaTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.testing;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+class DataSetRowMetaTest {
+
+ @BeforeAll
+ static void initHop() throws HopException {
+ HopEnvironment.init();
+ }
+
+ @Test
+ void createFieldsFromRowMetaCopiesNameTypeLengthPrecisionFormat() {
+ IRowMeta rowMeta = new RowMeta();
+ ValueMetaString name = new ValueMetaString("name", 50, -1);
+ name.setComments("customer name");
+ name.setConversionMask(null);
+ rowMeta.addValueMeta(name);
+ ValueMetaInteger id = new ValueMetaInteger("id", 9, 0);
+ id.setConversionMask("0");
+ rowMeta.addValueMeta(id);
+
+ List<DataSetField> fields = DataSet.createFieldsFromRowMeta(rowMeta);
+
+ assertEquals(2, fields.size());
+ assertEquals("name", fields.get(0).getFieldName());
+ assertEquals(name.getType(), fields.get(0).getType());
+ assertEquals(50, fields.get(0).getLength());
+ assertEquals("customer name", fields.get(0).getComment());
+ assertEquals("id", fields.get(1).getFieldName());
+ assertEquals(id.getType(), fields.get(1).getType());
+ assertEquals("0", fields.get(1).getFormat());
+ }
+
+ @Test
+ void validateRowMetaAcceptsMatchingLayout() throws HopException {
+ DataSet dataSet = sampleDataSet();
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaInteger("id"));
+ input.addValueMeta(new ValueMetaString("name"));
+ dataSet.validateRowMeta(input);
+ }
+
+ @Test
+ void validateRowMetaAcceptsCaseInsensitiveNames() throws HopException {
+ DataSet dataSet = sampleDataSet();
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaInteger("ID"));
+ input.addValueMeta(new ValueMetaString("NAME"));
+ dataSet.validateRowMeta(input);
+ }
+
+ @Test
+ void validateRowMetaRejectsTypeMismatch() {
+ DataSet dataSet = sampleDataSet();
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaString("id"));
+ input.addValueMeta(new ValueMetaString("name"));
+ HopException exception = assertThrows(HopException.class, () ->
dataSet.validateRowMeta(input));
+ assertTrue(exception.getMessage().contains("type"));
+ }
+
+ @Test
+ void validateRowMetaRejectsNameAndCountMismatch() {
+ DataSet dataSet = sampleDataSet();
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaInteger("id"));
+ HopException exception = assertThrows(HopException.class, () ->
dataSet.validateRowMeta(input));
+ assertTrue(exception.getMessage().contains("field count"));
+ }
+
+ private DataSet sampleDataSet() {
+ DataSet dataSet = new DataSet();
+ dataSet.setName("customers");
+ dataSet.setFields(
+ List.of(
+ new DataSetField("id", IValueMeta.TYPE_INTEGER, 9, 0, "", "0"),
+ new DataSetField("name", IValueMeta.TYPE_STRING, 50, -1, "", "")));
+ return dataSet;
+ }
+}
diff --git
a/plugins/misc/testing/src/test/java/org/apache/hop/testing/transforms/datasetoutput/DataSetOutputMetaTest.java
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/transforms/datasetoutput/DataSetOutputMetaTest.java
new file mode 100644
index 0000000000..013d8b0ee3
--- /dev/null
+++
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/transforms/datasetoutput/DataSetOutputMetaTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.testing.transforms.datasetoutput;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.hop.pipeline.transform.TransformSerializationTestUtil;
+import org.junit.jupiter.api.Test;
+
+class DataSetOutputMetaTest {
+
+ @Test
+ void testSerialization() throws Exception {
+ DataSetOutputMeta meta =
+ TransformSerializationTestUtil.testSerialization(
+ "/data-set-output-transform.xml", DataSetOutputMeta.class);
+
+ assertEquals("golden-customers", meta.getDataSetName());
+ assertEquals("${HOP_DATASETS_FOLDER}", meta.getFolderName());
+ assertEquals("golden-customers.csv", meta.getCsvFilename());
+ assertTrue(meta.isRecreateDataSet());
+ assertFalse(meta.isValidateDataSet());
+ }
+
+ @Test
+ void testSetDefault() {
+ DataSetOutputMeta meta = new DataSetOutputMeta();
+ meta.setRecreateDataSet(false);
+ meta.setValidateDataSet(true);
+ meta.setDefault();
+ assertTrue(meta.isRecreateDataSet());
+ assertFalse(meta.isValidateDataSet());
+ }
+}
diff --git
a/plugins/misc/testing/src/test/resources/data-set-output-transform.xml
b/plugins/misc/testing/src/test/resources/data-set-output-transform.xml
new file mode 100644
index 0000000000..b64648d3f6
--- /dev/null
+++ b/plugins/misc/testing/src/test/resources/data-set-output-transform.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+<transform>
+ <name>Data set output</name>
+ <type>DataSetOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <dataSetName>golden-customers</dataSetName>
+ <folderName>${HOP_DATASETS_FOLDER}</folderName>
+ <csvFilename>golden-customers.csv</csvFilename>
+ <recreateDataSet>Y</recreateDataSet>
+ <validateDataSet>N</validateDataSet>
+ <attributes/>
+ <GUI>
+ <xloc>240</xloc>
+ <yloc>96</yloc>
+ </GUI>
+</transform>
