This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 5fc8057c7f Improve YAML Input with nested JsonPath expressions (#7432)
5fc8057c7f is described below
commit 5fc8057c7f84a19e8d421bf5fb677875dcbd5e20
Author: Lance <[email protected]>
AuthorDate: Thu Jul 9 18:43:17 2026 +0800
Improve YAML Input with nested JsonPath expressions (#7432)
* Improve YAML Input with nested JsonPath expressions
Signed-off-by: lance <[email protected]>
* Improve YAML Input with nested JsonPath expressions
Signed-off-by: lance <[email protected]>
* add license
Signed-off-by: lance <[email protected]>
* add some tests
---------
Signed-off-by: lance <[email protected]>
Co-authored-by: Hans Van Akelyen <[email protected]>
---
...nput-validate.hpl => 0088-yaml-input-field.hpl} | 170 +++++++------
...input-validate.hpl => 0088-yaml-input-list.hpl} | 88 +------
...alidate.hpl => 0088-yaml-input-multi-level.hpl} | 167 +++++-------
...t-validate.hpl => 0088-yaml-input-multidoc.hpl} | 72 +-----
.../transforms/0088-yaml-input-validate.hpl | 139 +++++-----
.../datasets/golden-yaml-input-field.csv | 3 +
.../transforms/datasets/golden-yaml-input-list.csv | 6 +
.../datasets/golden-yaml-input-multi-level.csv | 6 +
.../datasets/golden-yaml-input-multidoc.csv | 4 +
.../transforms/datasets/golden-yaml-input.csv | 6 +-
.../transforms/files/yaml-input-list.yaml | 25 ++
.../transforms/files/yaml-input-multi-level.yaml | 35 +++
.../transforms/files/yaml-input-multidoc.yaml | 31 +++
...088-yaml-input.hwf => main-0088-yaml-input.hwf} | 11 +-
.../metadata/dataset/golden-yaml-input-field.json | 32 +++
.../metadata/dataset/golden-yaml-input-list.json | 16 ++
.../dataset/golden-yaml-input-multi-level.json | 32 +++
.../dataset/golden-yaml-input-multidoc.json | 32 +++
.../0088-yaml-input-field-validate UNIT.json | 38 +++
.../0088-yaml-input-list-validate UNIT.json | 28 +++
.../0088-yaml-input-multi-level UNIT.json | 38 +++
.../0088-yaml-input-multidoc-validate UNIT.json | 38 +++
plugins/transforms/yamlinput/pom.xml | 6 +
.../pipeline/transforms/yamlinput/YamlInput.java | 17 +-
.../transforms/yamlinput/YamlInputData.java | 4 +
.../transforms/yamlinput/YamlInputDialog.java | 6 +-
.../transforms/yamlinput/YamlPathResolver.java | 64 +++++
.../pipeline/transforms/yamlinput/YamlReader.java | 280 ++++++++++++++++++---
.../yamlinput/messages/messages_en_US.properties | 6 +-
.../transforms/yamlinput/YamlInputTests.java | 3 +
.../yamlinput/YamlPathResolverTests.java | 71 ++++++
.../transforms/yamlinput/YamlReaderTests.java | 166 +++++++++++-
32 files changed, 1189 insertions(+), 451 deletions(-)
diff --git a/integration-tests/transforms/0088-yaml-input-validate.hpl
b/integration-tests/transforms/0088-yaml-input-field.hpl
similarity index 64%
copy from integration-tests/transforms/0088-yaml-input-validate.hpl
copy to integration-tests/transforms/0088-yaml-input-field.hpl
index 1c70c26701..493bc72040 100644
--- a/integration-tests/transforms/0088-yaml-input-validate.hpl
+++ b/integration-tests/transforms/0088-yaml-input-field.hpl
@@ -19,7 +19,7 @@ limitations under the License.
-->
<pipeline>
<info>
- <name>0088-yaml-input-validate</name>
+ <name>0088-yaml-input-field</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@@ -31,22 +31,32 @@ limitations under the License.
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
- <created_date>2026/03/24 20:36:32.371</created_date>
+ <created_date>2026/07/09 11:00:00.000</created_date>
<modified_user>-</modified_user>
- <modified_date>2026/03/24 20:36:32.371</modified_date>
+ <modified_date>2026/07/09 11:00:00.000</modified_date>
</info>
<notepads>
</notepads>
<order>
<hop>
- <from>files/yaml-input.yaml</from>
+ <from>generate yaml rows</from>
+ <to>parse yaml field</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>parse yaml field</from>
+ <to>keep extracted fields</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>keep extracted fields</from>
<to>validate</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
- <name>files/yaml-input.yaml</name>
- <type>YamlInput</type>
+ <name>generate yaml rows</name>
+ <type>DataGrid</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@@ -55,37 +65,20 @@ limitations under the License.
<method>none</method>
<schema_name/>
</partitioning>
- <include>N</include>
- <include_field/>
- <rownum>N</rownum>
- <addresultfile>N</addresultfile>
- <validating>N</validating>
- <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
- <doNotFailIfNoFile>Y</doNotFailIfNoFile>
- <rownum_field/>
- <encoding/>
- <file>
- <name>${PROJECT_HOME}/files/yaml-input.yaml</name>
- <filemask/>
- <file_required>N</file_required>
- <include_subfolders>N</include_subfolders>
- </file>
<fields>
<field>
- <name>organisation</name>
- <path>organisation</path>
- <type>String</type>
+ <name>id</name>
+ <type>Integer</type>
<format/>
<currency/>
<decimal/>
<group/>
<length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
+ <precision>0</precision>
+ <set_empty_string>N</set_empty_string>
</field>
<field>
- <name>acronym</name>
- <path>acronym</path>
+ <name>yaml</name>
<type>String</type>
<format/>
<currency/>
@@ -93,11 +86,54 @@ limitations under the License.
<group/>
<length>-1</length>
<precision>-1</precision>
- <trim_type>none</trim_type>
+ <set_empty_string>N</set_empty_string>
</field>
+ </fields>
+ <data>
+ <line>
+ <item>1</item>
+ <item>{name: Spark, version: "4.0.0"}</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>{name: Flink, version: "1.20"}</item>
+ </line>
+ </data>
+ <GUI>
+ <xloc>80</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>parse yaml field</name>
+ <type>YamlInput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <include>N</include>
+ <include_field/>
+ <rownum>N</rownum>
+ <addresultfile>N</addresultfile>
+ <validating>N</validating>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <rownum_field/>
+ <encoding/>
+ <file>
+ <name/>
+ <filemask/>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ <fields>
<field>
- <name>project</name>
- <path>project</path>
+ <name>name</name>
+ <path>name</path>
<type>String</type>
<format/>
<currency/>
@@ -108,9 +144,9 @@ limitations under the License.
<trim_type>none</trim_type>
</field>
<field>
- <name>javaFileCount</name>
- <path>javaFileCount</path>
- <type>Integer</type>
+ <name>version</name>
+ <path>version</path>
+ <type>String</type>
<format/>
<currency/>
<decimal/>
@@ -119,50 +155,42 @@ limitations under the License.
<precision>-1</precision>
<trim_type>none</trim_type>
</field>
+ </fields>
+ <limit>0</limit>
+ <IsInFields>Y</IsInFields>
+ <IsAFile>N</IsAFile>
+ <YamlField>yaml</YamlField>
+ <attributes/>
+ <GUI>
+ <xloc>272</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>keep extracted fields</name>
+ <type>SelectValues</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
<field>
- <name>awesome</name>
- <path>awesome</path>
- <type>Boolean</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
+ <name>id</name>
</field>
<field>
- <name>used-apache-projects</name>
- <path>used-apache-projects</path>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
+ <name>name</name>
</field>
<field>
- <name>statistics</name>
- <path>statistics</path>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
+ <name>version</name>
</field>
+ <select_unspecified>N</select_unspecified>
</fields>
- <limit>0</limit>
- <IsInFields>N</IsInFields>
- <IsAFile>N</IsAFile>
- <YamlField/>
- <attributes/>
<GUI>
- <xloc>80</xloc>
+ <xloc>464</xloc>
<yloc>64</yloc>
</GUI>
</transform>
@@ -179,7 +207,7 @@ limitations under the License.
</partitioning>
<attributes/>
<GUI>
- <xloc>288</xloc>
+ <xloc>656</xloc>
<yloc>64</yloc>
</GUI>
</transform>
diff --git a/integration-tests/transforms/0088-yaml-input-validate.hpl
b/integration-tests/transforms/0088-yaml-input-list.hpl
similarity index 58%
copy from integration-tests/transforms/0088-yaml-input-validate.hpl
copy to integration-tests/transforms/0088-yaml-input-list.hpl
index 1c70c26701..a3c4c4b2d7 100644
--- a/integration-tests/transforms/0088-yaml-input-validate.hpl
+++ b/integration-tests/transforms/0088-yaml-input-list.hpl
@@ -19,7 +19,7 @@ limitations under the License.
-->
<pipeline>
<info>
- <name>0088-yaml-input-validate</name>
+ <name>0088-yaml-input-list</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@@ -31,21 +31,21 @@ limitations under the License.
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
- <created_date>2026/03/24 20:36:32.371</created_date>
+ <created_date>2026/07/09 11:00:00.000</created_date>
<modified_user>-</modified_user>
- <modified_date>2026/03/24 20:36:32.371</modified_date>
+ <modified_date>2026/07/09 11:00:00.000</modified_date>
</info>
<notepads>
</notepads>
<order>
<hop>
- <from>files/yaml-input.yaml</from>
+ <from>files/yaml-input-list.yaml</from>
<to>validate</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
- <name>files/yaml-input.yaml</name>
+ <name>files/yaml-input-list.yaml</name>
<type>YamlInput</type>
<description/>
<distribute>Y</distribute>
@@ -65,87 +65,15 @@ limitations under the License.
<rownum_field/>
<encoding/>
<file>
- <name>${PROJECT_HOME}/files/yaml-input.yaml</name>
+ <name>${PROJECT_HOME}/files/yaml-input-list.yaml</name>
<filemask/>
<file_required>N</file_required>
<include_subfolders>N</include_subfolders>
</file>
<fields>
<field>
- <name>organisation</name>
- <path>organisation</path>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
- </field>
- <field>
- <name>acronym</name>
- <path>acronym</path>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
- </field>
- <field>
- <name>project</name>
- <path>project</path>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
- </field>
- <field>
- <name>javaFileCount</name>
- <path>javaFileCount</path>
- <type>Integer</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
- </field>
- <field>
- <name>awesome</name>
- <path>awesome</path>
- <type>Boolean</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
- </field>
- <field>
- <name>used-apache-projects</name>
- <path>used-apache-projects</path>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
- </field>
- <field>
- <name>statistics</name>
- <path>statistics</path>
+ <name>engine</name>
+ <path>engine</path>
<type>String</type>
<format/>
<currency/>
diff --git a/integration-tests/transforms/0088-yaml-input-validate.hpl
b/integration-tests/transforms/0088-yaml-input-multi-level.hpl
similarity index 58%
copy from integration-tests/transforms/0088-yaml-input-validate.hpl
copy to integration-tests/transforms/0088-yaml-input-multi-level.hpl
index 1c70c26701..73351ccfd1 100644
--- a/integration-tests/transforms/0088-yaml-input-validate.hpl
+++ b/integration-tests/transforms/0088-yaml-input-multi-level.hpl
@@ -19,171 +19,116 @@ limitations under the License.
-->
<pipeline>
<info>
- <name>0088-yaml-input-validate</name>
- <name_sync_with_filename>Y</name_sync_with_filename>
- <description/>
- <extended_description/>
- <pipeline_version/>
- <pipeline_type>Normal</pipeline_type>
- <parameters>
- </parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>-1</pipeline_status>
+ <parameters/>
+ <name>New pipeline</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
<created_user>-</created_user>
- <created_date>2026/03/24 20:36:32.371</created_date>
<modified_user>-</modified_user>
- <modified_date>2026/03/24 20:36:32.371</modified_date>
+ <created_date>2026/07/05 10:23:55.458</created_date>
+ <modified_date>2026/07/05 10:23:55.458</modified_date>
</info>
- <notepads>
- </notepads>
- <order>
- <hop>
- <from>files/yaml-input.yaml</from>
- <to>validate</to>
- <enabled>Y</enabled>
- </hop>
- </order>
<transform>
- <name>files/yaml-input.yaml</name>
<type>YamlInput</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
+ <name>files/yaml-input.yaml</name>
<include>N</include>
<include_field/>
<rownum>N</rownum>
+ <rownum_field/>
+ <limit>0</limit>
+ <encoding/>
+ <YamlField/>
+ <IsInFields>N</IsInFields>
+ <IsAFile>N</IsAFile>
<addresultfile>N</addresultfile>
<validating>N</validating>
<IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
<doNotFailIfNoFile>Y</doNotFailIfNoFile>
- <rownum_field/>
- <encoding/>
- <file>
- <name>${PROJECT_HOME}/files/yaml-input.yaml</name>
- <filemask/>
- <file_required>N</file_required>
- <include_subfolders>N</include_subfolders>
- </file>
+ <files>
+ <file>
+ <name>${PROJECT_HOME}/files/yaml-input-multi-level.yaml</name>
+ <filemask/>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ </files>
<fields>
<field>
- <name>organisation</name>
- <path>organisation</path>
+ <name>tableName</name>
+ <path>$.tables[*].tableName</path>
<type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
<length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
- </field>
- <field>
- <name>acronym</name>
- <path>acronym</path>
- <type>String</type>
<format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
<trim_type>none</trim_type>
- </field>
- <field>
- <name>project</name>
- <path>project</path>
- <type>String</type>
- <format/>
+ <precision>-1</precision>
<currency/>
<decimal/>
<group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
</field>
<field>
- <name>javaFileCount</name>
- <path>javaFileCount</path>
+ <name>dataPagination</name>
+ <path>$.tables[*].dataPagination</path>
<type>Integer</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
<length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
- </field>
- <field>
- <name>awesome</name>
- <path>awesome</path>
- <type>Boolean</type>
<format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
<trim_type>none</trim_type>
- </field>
- <field>
- <name>used-apache-projects</name>
- <path>used-apache-projects</path>
- <type>String</type>
- <format/>
+ <precision>-1</precision>
<currency/>
<decimal/>
<group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
</field>
<field>
- <name>statistics</name>
- <path>statistics</path>
+ <name>queryDataset</name>
+ <path>$.tables[*].queryDataset</path>
<type>String</type>
+ <length>-1</length>
<format/>
+ <trim_type>none</trim_type>
+ <precision>-1</precision>
<currency/>
<decimal/>
<group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
</field>
</fields>
- <limit>0</limit>
- <IsInFields>N</IsInFields>
- <IsAFile>N</IsAFile>
- <YamlField/>
- <attributes/>
+ <distribute>N</distribute>
+ <copies>1</copies>
<GUI>
- <xloc>80</xloc>
- <yloc>64</yloc>
+ <xloc>96</xloc>
+ <yloc>48</yloc>
</GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
</transform>
<transform>
- <name>validate</name>
<type>Dummy</type>
- <description/>
+ <name>validate</name>
<distribute>Y</distribute>
- <custom_distribution/>
<copies>1</copies>
+ <GUI>
+ <xloc>320</xloc>
+ <yloc>48</yloc>
+ </GUI>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
- <GUI>
- <xloc>288</xloc>
- <yloc>64</yloc>
- </GUI>
</transform>
- <transform_error_handling>
- </transform_error_handling>
+ <order>
+ <hop>
+ <from>files/yaml-input.yaml</from>
+ <to>validate</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <notepads/>
<attributes/>
+ <transform_error_handling/>
</pipeline>
diff --git a/integration-tests/transforms/0088-yaml-input-validate.hpl
b/integration-tests/transforms/0088-yaml-input-multidoc.hpl
similarity index 67%
copy from integration-tests/transforms/0088-yaml-input-validate.hpl
copy to integration-tests/transforms/0088-yaml-input-multidoc.hpl
index 1c70c26701..ee18141400 100644
--- a/integration-tests/transforms/0088-yaml-input-validate.hpl
+++ b/integration-tests/transforms/0088-yaml-input-multidoc.hpl
@@ -19,7 +19,7 @@ limitations under the License.
-->
<pipeline>
<info>
- <name>0088-yaml-input-validate</name>
+ <name>0088-yaml-input-multidoc</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@@ -31,21 +31,21 @@ limitations under the License.
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
- <created_date>2026/03/24 20:36:32.371</created_date>
+ <created_date>2026/07/09 11:00:00.000</created_date>
<modified_user>-</modified_user>
- <modified_date>2026/03/24 20:36:32.371</modified_date>
+ <modified_date>2026/07/09 11:00:00.000</modified_date>
</info>
<notepads>
</notepads>
<order>
<hop>
- <from>files/yaml-input.yaml</from>
+ <from>files/yaml-input-multidoc.yaml</from>
<to>validate</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
- <name>files/yaml-input.yaml</name>
+ <name>files/yaml-input-multidoc.yaml</name>
<type>YamlInput</type>
<description/>
<distribute>Y</distribute>
@@ -65,15 +65,15 @@ limitations under the License.
<rownum_field/>
<encoding/>
<file>
- <name>${PROJECT_HOME}/files/yaml-input.yaml</name>
+ <name>${PROJECT_HOME}/files/yaml-input-multidoc.yaml</name>
<filemask/>
<file_required>N</file_required>
<include_subfolders>N</include_subfolders>
</file>
<fields>
<field>
- <name>organisation</name>
- <path>organisation</path>
+ <name>name</name>
+ <path>name</path>
<type>String</type>
<format/>
<currency/>
@@ -84,8 +84,8 @@ limitations under the License.
<trim_type>none</trim_type>
</field>
<field>
- <name>acronym</name>
- <path>acronym</path>
+ <name>type</name>
+ <path>type</path>
<type>String</type>
<format/>
<currency/>
@@ -96,32 +96,8 @@ limitations under the License.
<trim_type>none</trim_type>
</field>
<field>
- <name>project</name>
- <path>project</path>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
- </field>
- <field>
- <name>javaFileCount</name>
- <path>javaFileCount</path>
- <type>Integer</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
- </field>
- <field>
- <name>awesome</name>
- <path>awesome</path>
+ <name>distributed</name>
+ <path>distributed</path>
<type>Boolean</type>
<format/>
<currency/>
@@ -131,30 +107,6 @@ limitations under the License.
<precision>-1</precision>
<trim_type>none</trim_type>
</field>
- <field>
- <name>used-apache-projects</name>
- <path>used-apache-projects</path>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
- </field>
- <field>
- <name>statistics</name>
- <path>statistics</path>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
- </field>
</fields>
<limit>0</limit>
<IsInFields>N</IsInFields>
diff --git a/integration-tests/transforms/0088-yaml-input-validate.hpl
b/integration-tests/transforms/0088-yaml-input-validate.hpl
index 1c70c26701..8ccc00fff5 100644
--- a/integration-tests/transforms/0088-yaml-input-validate.hpl
+++ b/integration-tests/transforms/0088-yaml-input-validate.hpl
@@ -19,171 +19,168 @@ limitations under the License.
-->
<pipeline>
<info>
- <name>0088-yaml-input-validate</name>
- <name_sync_with_filename>Y</name_sync_with_filename>
- <description/>
- <extended_description/>
<pipeline_version/>
- <pipeline_type>Normal</pipeline_type>
- <parameters>
- </parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters/>
+ <name>0088-yaml-input-validate</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
<created_user>-</created_user>
- <created_date>2026/03/24 20:36:32.371</created_date>
<modified_user>-</modified_user>
+ <created_date>2026/03/24 20:36:32.371</created_date>
<modified_date>2026/03/24 20:36:32.371</modified_date>
</info>
- <notepads>
- </notepads>
- <order>
- <hop>
- <from>files/yaml-input.yaml</from>
- <to>validate</to>
- <enabled>Y</enabled>
- </hop>
- </order>
<transform>
- <name>files/yaml-input.yaml</name>
<type>YamlInput</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
+ <name>files/yaml-input.yaml</name>
<include>N</include>
<include_field/>
<rownum>N</rownum>
+ <rownum_field/>
+ <limit>0</limit>
+ <encoding/>
+ <YamlField/>
+ <IsInFields>N</IsInFields>
+ <IsAFile>N</IsAFile>
<addresultfile>N</addresultfile>
<validating>N</validating>
<IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
<doNotFailIfNoFile>Y</doNotFailIfNoFile>
- <rownum_field/>
- <encoding/>
- <file>
- <name>${PROJECT_HOME}/files/yaml-input.yaml</name>
- <filemask/>
- <file_required>N</file_required>
- <include_subfolders>N</include_subfolders>
- </file>
+ <files>
+ <file>
+ <name>${PROJECT_HOME}/files/yaml-input.yaml</name>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ </files>
<fields>
<field>
<name>organisation</name>
<path>organisation</path>
<type>String</type>
+ <length>-1</length>
<format/>
+ <trim_type>none</trim_type>
+ <precision>-1</precision>
<currency/>
<decimal/>
<group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
</field>
<field>
<name>acronym</name>
<path>acronym</path>
<type>String</type>
+ <length>-1</length>
<format/>
+ <trim_type>none</trim_type>
+ <precision>-1</precision>
<currency/>
<decimal/>
<group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
</field>
<field>
<name>project</name>
<path>project</path>
<type>String</type>
+ <length>-1</length>
<format/>
+ <trim_type>none</trim_type>
+ <precision>-1</precision>
<currency/>
<decimal/>
<group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
</field>
<field>
<name>javaFileCount</name>
<path>javaFileCount</path>
<type>Integer</type>
+ <length>-1</length>
<format/>
+ <trim_type>none</trim_type>
+ <precision>-1</precision>
<currency/>
<decimal/>
<group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
</field>
<field>
<name>awesome</name>
<path>awesome</path>
<type>Boolean</type>
+ <length>-1</length>
<format/>
+ <trim_type>none</trim_type>
+ <precision>-1</precision>
<currency/>
<decimal/>
<group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
</field>
<field>
<name>used-apache-projects</name>
<path>used-apache-projects</path>
<type>String</type>
+ <length>-1</length>
<format/>
+ <trim_type>none</trim_type>
+ <precision>-1</precision>
<currency/>
<decimal/>
<group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
</field>
<field>
<name>statistics</name>
<path>statistics</path>
<type>String</type>
+ <length>-1</length>
<format/>
+ <trim_type>none</trim_type>
+ <precision>-1</precision>
<currency/>
<decimal/>
<group/>
- <length>-1</length>
- <precision>-1</precision>
- <trim_type>none</trim_type>
</field>
</fields>
- <limit>0</limit>
- <IsInFields>N</IsInFields>
- <IsAFile>N</IsAFile>
- <YamlField/>
- <attributes/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
<GUI>
- <xloc>80</xloc>
- <yloc>64</yloc>
+ <xloc>48</xloc>
+ <yloc>48</yloc>
</GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
</transform>
<transform>
- <name>validate</name>
<type>Dummy</type>
- <description/>
+ <name>validate</name>
<distribute>Y</distribute>
- <custom_distribution/>
<copies>1</copies>
+ <GUI>
+ <xloc>192</xloc>
+ <yloc>48</yloc>
+ </GUI>
+ <description/>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
- <GUI>
- <xloc>288</xloc>
- <yloc>64</yloc>
- </GUI>
</transform>
- <transform_error_handling>
- </transform_error_handling>
+ <order>
+ <hop>
+ <from>files/yaml-input.yaml</from>
+ <to>validate</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <notepads/>
<attributes/>
+ <transform_error_handling/>
</pipeline>
diff --git a/integration-tests/transforms/datasets/golden-yaml-input-field.csv
b/integration-tests/transforms/datasets/golden-yaml-input-field.csv
new file mode 100644
index 0000000000..bbc16a5dc0
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-yaml-input-field.csv
@@ -0,0 +1,3 @@
+id,name,version
+1,Spark,4.0.0
+2,Flink,1.20
diff --git a/integration-tests/transforms/datasets/golden-yaml-input-list.csv
b/integration-tests/transforms/datasets/golden-yaml-input-list.csv
new file mode 100644
index 0000000000..fe705a8965
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-yaml-input-list.csv
@@ -0,0 +1,6 @@
+engine
+Spark
+Flink
+Beam
+Commons
+VFS
diff --git
a/integration-tests/transforms/datasets/golden-yaml-input-multi-level.csv
b/integration-tests/transforms/datasets/golden-yaml-input-multi-level.csv
new file mode 100644
index 0000000000..879fb44f21
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-yaml-input-multi-level.csv
@@ -0,0 +1,6 @@
+tableName,dataPagination,queryDataset
+t_user,100,"SELECT * FROM crm.t_user
+"
+t_customer,20,"SELECT * FROM crm.t_customer
+"
+t_order,30,SELECT * FROM crm.t_order
diff --git
a/integration-tests/transforms/datasets/golden-yaml-input-multidoc.csv
b/integration-tests/transforms/datasets/golden-yaml-input-multidoc.csv
new file mode 100644
index 0000000000..2f35075692
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-yaml-input-multidoc.csv
@@ -0,0 +1,4 @@
+name,type,distributed
+Spark,engine,Y
+Flink,engine,Y
+Janino,compiler,N
diff --git a/integration-tests/transforms/datasets/golden-yaml-input.csv
b/integration-tests/transforms/datasets/golden-yaml-input.csv
index 6a7bac4601..84137e344f 100644
--- a/integration-tests/transforms/datasets/golden-yaml-input.csv
+++ b/integration-tests/transforms/datasets/golden-yaml-input.csv
@@ -1,2 +1,6 @@
organisation,acronym,project,javaFileCount,awesome,used-apache-projects,statistics
-The Apache Software Foundation,The ASF,Apache Hop,4447,Y,"[Spark, Flink, Beam,
Commons, VFS]","{number-of-libs: 1159,number-of-messages-bundles:
13025,integration-test-projects: {count: 39,pipelines: 1032,workflows:
498}}"
+The Apache Software Foundation,The ASF,Apache
Hop,4447,Y,Spark,"{number-of-libs: 1159,number-of-messages-bundles:
13025,integration-test-projects: {count: 39,pipelines: 1032,workflows:
498}}"
+The Apache Software Foundation,The ASF,Apache
Hop,4447,Y,Flink,"{number-of-libs: 1159,number-of-messages-bundles:
13025,integration-test-projects: {count: 39,pipelines: 1032,workflows:
498}}"
+The Apache Software Foundation,The ASF,Apache
Hop,4447,Y,Beam,"{number-of-libs: 1159,number-of-messages-bundles:
13025,integration-test-projects: {count: 39,pipelines: 1032,workflows:
498}}"
+The Apache Software Foundation,The ASF,Apache
Hop,4447,Y,Commons,"{number-of-libs: 1159,number-of-messages-bundles:
13025,integration-test-projects: {count: 39,pipelines: 1032,workflows:
498}}"
+The Apache Software Foundation,The ASF,Apache Hop,4447,Y,VFS,"{number-of-libs:
1159,number-of-messages-bundles: 13025,integration-test-projects: {count:
39,pipelines: 1032,workflows: 498}}"
diff --git a/integration-tests/transforms/files/yaml-input-list.yaml
b/integration-tests/transforms/files/yaml-input-list.yaml
new file mode 100644
index 0000000000..48b5ab0370
--- /dev/null
+++ b/integration-tests/transforms/files/yaml-input-list.yaml
@@ -0,0 +1,25 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+---
+ - Spark
+ - Flink
+ - Beam
+ - Commons
+ - VFS
diff --git a/integration-tests/transforms/files/yaml-input-multi-level.yaml
b/integration-tests/transforms/files/yaml-input-multi-level.yaml
new file mode 100644
index 0000000000..875861fb35
--- /dev/null
+++ b/integration-tests/transforms/files/yaml-input-multi-level.yaml
@@ -0,0 +1,35 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+
+tables:
+ - tableName: t_user
+ dataPagination: 100
+ queryDataset: |
+ SELECT * FROM crm.t_user
+
+ - tableName: t_customer
+ dataPagination: 20
+ queryDataset: |
+ SELECT * FROM crm.t_customer
+
+ - tableName: t_order
+ dataPagination: 30
+ queryDataset: |
+ SELECT * FROM crm.t_order
\ No newline at end of file
diff --git a/integration-tests/transforms/files/yaml-input-multidoc.yaml
b/integration-tests/transforms/files/yaml-input-multidoc.yaml
new file mode 100644
index 0000000000..e9436455c8
--- /dev/null
+++ b/integration-tests/transforms/files/yaml-input-multidoc.yaml
@@ -0,0 +1,31 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+---
+ name: "Spark"
+ type: "engine"
+ distributed: true
+---
+ name: "Flink"
+ type: "engine"
+ distributed: true
+---
+ name: "Janino"
+ type: "compiler"
+ distributed: false
diff --git a/integration-tests/transforms/0088-yaml-input.hwf
b/integration-tests/transforms/main-0088-yaml-input.hwf
similarity index 87%
rename from integration-tests/transforms/0088-yaml-input.hwf
rename to integration-tests/transforms/main-0088-yaml-input.hwf
index 4440be7452..1c9bb28beb 100644
--- a/integration-tests/transforms/0088-yaml-input.hwf
+++ b/integration-tests/transforms/main-0088-yaml-input.hwf
@@ -18,7 +18,7 @@ limitations under the License.
-->
<workflow>
- <name>0088-yaml-input</name>
+ <name>main-0088-yaml-input</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@@ -58,6 +58,15 @@ limitations under the License.
<test_name>
<name>0088-yaml-input-validate UNIT</name>
</test_name>
+ <test_name>
+ <name>0088-yaml-input-list-validate UNIT</name>
+ </test_name>
+ <test_name>
+ <name>0088-yaml-input-multidoc-validate UNIT</name>
+ </test_name>
+ <test_name>
+ <name>0088-yaml-input-field-validate UNIT</name>
+ </test_name>
</test_names>
<parallel>N</parallel>
<xloc>272</xloc>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-yaml-input-field.json
b/integration-tests/transforms/metadata/dataset/golden-yaml-input-field.json
new file mode 100644
index 0000000000..9a49b66009
--- /dev/null
+++ b/integration-tests/transforms/metadata/dataset/golden-yaml-input-field.json
@@ -0,0 +1,32 @@
+{
+ "base_filename": "golden-yaml-input-field.csv",
+ "name": "golden-yaml-input-field",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "id",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "version",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
diff --git
a/integration-tests/transforms/metadata/dataset/golden-yaml-input-list.json
b/integration-tests/transforms/metadata/dataset/golden-yaml-input-list.json
new file mode 100644
index 0000000000..a1b9b54a58
--- /dev/null
+++ b/integration-tests/transforms/metadata/dataset/golden-yaml-input-list.json
@@ -0,0 +1,16 @@
+{
+ "base_filename": "golden-yaml-input-list.csv",
+ "name": "golden-yaml-input-list",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "engine",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
diff --git
a/integration-tests/transforms/metadata/dataset/golden-yaml-input-multi-level.json
b/integration-tests/transforms/metadata/dataset/golden-yaml-input-multi-level.json
new file mode 100644
index 0000000000..4b6827952d
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-yaml-input-multi-level.json
@@ -0,0 +1,32 @@
+{
+ "base_filename": "golden-yaml-input-multi-level.csv",
+ "name": "golden-yaml-input-multi-level",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "tableName",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "dataPagination",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "queryDataset",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/dataset/golden-yaml-input-multidoc.json
b/integration-tests/transforms/metadata/dataset/golden-yaml-input-multidoc.json
new file mode 100644
index 0000000000..94463d76fc
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-yaml-input-multidoc.json
@@ -0,0 +1,32 @@
+{
+ "base_filename": "golden-yaml-input-multidoc.csv",
+ "name": "golden-yaml-input-multidoc",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "type",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 4,
+ "field_precision": -1,
+ "field_name": "distributed",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
diff --git
a/integration-tests/transforms/metadata/unit-test/0088-yaml-input-field-validate
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0088-yaml-input-field-validate
UNIT.json
new file mode 100644
index 0000000000..7e72d059cd
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0088-yaml-input-field-validate
UNIT.json
@@ -0,0 +1,38 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "name",
+ "data_set_field": "name"
+ },
+ {
+ "transform_field": "version",
+ "data_set_field": "version"
+ }
+ ],
+ "field_order": [
+ "id",
+ "name",
+ "version"
+ ],
+ "data_set_name": "golden-yaml-input-field",
+ "transform_name": "validate"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0088-yaml-input-field-validate UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0088-yaml-input-field.hpl"
+}
diff --git
a/integration-tests/transforms/metadata/unit-test/0088-yaml-input-list-validate
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0088-yaml-input-list-validate
UNIT.json
new file mode 100644
index 0000000000..b0cbd9976e
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0088-yaml-input-list-validate
UNIT.json
@@ -0,0 +1,28 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "engine",
+ "data_set_field": "engine"
+ }
+ ],
+ "field_order": [
+ "engine"
+ ],
+ "data_set_name": "golden-yaml-input-list",
+ "transform_name": "validate"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0088-yaml-input-list-validate UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0088-yaml-input-list.hpl"
+}
diff --git
a/integration-tests/transforms/metadata/unit-test/0088-yaml-input-multi-level
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0088-yaml-input-multi-level
UNIT.json
new file mode 100644
index 0000000000..4ad218273d
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0088-yaml-input-multi-level
UNIT.json
@@ -0,0 +1,38 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "tableName",
+ "data_set_field": "tableName"
+ },
+ {
+ "transform_field": "dataPagination",
+ "data_set_field": "dataPagination"
+ },
+ {
+ "transform_field": "queryDataset",
+ "data_set_field": "queryDataset"
+ }
+ ],
+ "field_order": [
+ "tableName",
+ "dataPagination",
+ "queryDataset"
+ ],
+ "data_set_name": "golden-yaml-input-multi-level",
+ "transform_name": "validate"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0088-yaml-input-multi-level UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0088-yaml-input-multi-level.hpl"
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/unit-test/0088-yaml-input-multidoc-validate
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0088-yaml-input-multidoc-validate
UNIT.json
new file mode 100644
index 0000000000..c5d7fed07a
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0088-yaml-input-multidoc-validate
UNIT.json
@@ -0,0 +1,38 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "name",
+ "data_set_field": "name"
+ },
+ {
+ "transform_field": "type",
+ "data_set_field": "type"
+ },
+ {
+ "transform_field": "distributed",
+ "data_set_field": "distributed"
+ }
+ ],
+ "field_order": [
+ "name",
+ "type",
+ "distributed"
+ ],
+ "data_set_name": "golden-yaml-input-multidoc",
+ "transform_name": "validate"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0088-yaml-input-multidoc-validate UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0088-yaml-input-multidoc.hpl"
+}
diff --git a/plugins/transforms/yamlinput/pom.xml
b/plugins/transforms/yamlinput/pom.xml
index b365b45faa..b7233803d9 100644
--- a/plugins/transforms/yamlinput/pom.xml
+++ b/plugins/transforms/yamlinput/pom.xml
@@ -42,6 +42,12 @@
</dependencyManagement>
<dependencies>
+ <dependency>
+ <groupId>com.jayway.jsonpath</groupId>
+ <artifactId>json-path</artifactId>
+ <version>${json-path.version}</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
diff --git
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java
index acb70b1f2a..93a7cfe0cc 100644
---
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java
+++
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java
@@ -132,12 +132,12 @@ public class YamlInput extends
BaseTransform<YamlInputMeta, YamlInputData> {
PKG, "YamlInput.Log.YAMLStream", meta.getYamlField(),
fieldValue));
}
- if (meta.isSourceFile()) {
-
- // source is a file.
+ data.yaml = new YamlReader();
+ data.yaml.setFieldPaths(data.fieldPaths);
+ // source is a file.
+ if (meta.isSourceFile()) {
FileObject yamlFile = HopVfs.getFileObject(fieldValue, variables);
- data.yaml = new YamlReader();
data.yaml.loadFile(yamlFile);
dataVolumeIn =
(dataVolumeIn != null ? dataVolumeIn : 0L) +
data.yaml.getBytesReadFromFile();
@@ -145,7 +145,6 @@ public class YamlInput extends BaseTransform<YamlInputMeta,
YamlInputData> {
addFileToResultFilesName(data.yaml.getFile());
} else {
- data.yaml = new YamlReader();
data.yaml.loadString(fieldValue);
}
} catch (Exception e) {
@@ -235,6 +234,7 @@ public class YamlInput extends BaseTransform<YamlInputMeta,
YamlInputData> {
// We have a file
// define a YAML reader and load file
data.yaml = new YamlReader();
+ data.yaml.setFieldPaths(data.fieldPaths);
data.yaml.loadFile(data.file);
dataVolumeIn =
(dataVolumeIn != null ? dataVolumeIn : 0L) +
data.yaml.getBytesReadFromFile();
@@ -416,14 +416,17 @@ public class YamlInput extends
BaseTransform<YamlInputMeta, YamlInputData> {
if (super.init()) {
data.rowIndex = 1L;
data.nrInputFields = meta.getInputFields().size();
+ data.fieldPaths = new String[data.nrInputFields];
data.rowMeta = new RowMeta();
for (int i = 0; i < data.nrInputFields; i++) {
YamlInputField field = meta.getInputFields().get(i);
- String path = resolve(field.getPath());
+ String fieldName = resolve(field.getName());
+ // resolve config yaml field path(.eg: $.table.name)
+ data.fieldPaths[i] = YamlPathResolver.resolvePath(this,
field.getPath());
try {
- IValueMeta valueMeta = ValueMetaFactory.createValueMeta(path,
field.getType());
+ IValueMeta valueMeta = ValueMetaFactory.createValueMeta(fieldName,
field.getType());
valueMeta.setTrimType(field.getTrimType());
data.rowMeta.addValueMeta(valueMeta);
} catch (Exception e) {
diff --git
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputData.java
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputData.java
index 0b53fbad96..7a798bb2b6 100644
---
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputData.java
+++
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputData.java
@@ -22,6 +22,7 @@ import org.apache.hop.core.fileinput.FileInputList;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.pipeline.transform.BaseTransformData;
+/** yaml input data */
@SuppressWarnings("java:S1104")
public class YamlInputData extends BaseTransformData {
public IRowMeta outputRowMeta;
@@ -45,6 +46,9 @@ public class YamlInputData extends BaseTransformData {
public IRowMeta rowMeta;
+ /** yaml field path[] */
+ public String[] fieldPaths;
+
public YamlInputData() {
super();
diff --git
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDialog.java
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDialog.java
index 8dbf1b40fb..b2aacde490 100644
---
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDialog.java
+++
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDialog.java
@@ -17,6 +17,7 @@
package org.apache.hop.pipeline.transforms.yamlinput;
+import java.util.List;
import org.apache.hop.core.Const;
import org.apache.hop.core.Props;
import org.apache.hop.core.exception.HopException;
@@ -993,13 +994,14 @@ public class YamlInputDialog extends BaseTransformDialog {
yaml = new YamlReader();
yaml.loadFile(inputList.getFile(0));
RowMeta row = yaml.getFields();
+ List<String> paths = yaml.getDiscoveredPaths();
- for (int i = 0; i < row.size(); i++) {
+ for (int i = 0; i < paths.size(); i++) {
IValueMeta value = row.getValueMeta(i);
TableItem item = new TableItem(wFields.table, SWT.NONE);
item.setText(1, value.getName());
- item.setText(2, value.getName());
+ item.setText(2, paths.get(i));
item.setText(3, value.getTypeDesc());
}
wFields.removeEmptyRows();
diff --git
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlPathResolver.java
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlPathResolver.java
new file mode 100644
index 0000000000..6b1b0253dc
--- /dev/null
+++
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlPathResolver.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.yamlinput;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.hop.core.util.StringUtil;
+import org.apache.hop.core.util.Utils;
+import org.apache.hop.core.variables.IVariables;
+
+/** Normalizes and resolves YAML/JSON-style path expressions for JsonPath
evaluation. */
+public final class YamlPathResolver {
+ public static final String DOLLAR = "$";
+
+ private YamlPathResolver() {}
+
+ /**
+ * Turns a legacy flat key or dotted path into a JsonPath expression. Paths
that already start
+ * with {@code $} are returned unchanged.
+ */
+ public static String normalize(String path) {
+ if (Utils.isEmpty(path)) {
+ return DOLLAR;
+ }
+
+ if (path.startsWith(DOLLAR)) {
+ return path;
+ }
+ return "$." + path;
+ }
+
+ /**
+ * Substitutes pipeline variables in a path without mangling JsonPath syntax
such as {@code $[*]}.
+ */
+ public static String resolvePath(IVariables variables, String path) {
+ if (Utils.isEmpty(path) || variables == null) {
+ return normalize(path);
+ }
+
+ Map<String, String> variableMap = new HashMap<>();
+ for (String name : variables.getVariableNames()) {
+ variableMap.put(name, variables.getVariable(name));
+ }
+
+ String substituted =
+ StringUtil.substituteUnix(StringUtil.substituteWindows(path,
variableMap), variableMap);
+ return normalize(substituted);
+ }
+}
diff --git
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReader.java
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReader.java
index d0acefd6da..6fba1d6108 100644
---
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReader.java
+++
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReader.java
@@ -17,14 +17,22 @@
package org.apache.hop.pipeline.transforms.yamlinput;
+import com.jayway.jsonpath.Configuration;
+import com.jayway.jsonpath.JsonPath;
+import com.jayway.jsonpath.Option;
+import com.jayway.jsonpath.ReadContext;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.Timestamp;
+import java.util.ArrayDeque;
import java.util.ArrayList;
+import java.util.Deque;
import java.util.Iterator;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import lombok.Getter;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.vfs2.FileObject;
@@ -47,6 +55,10 @@ import org.yaml.snakeyaml.Yaml;
public class YamlReader {
private static final String DEFAULT_LIST_VALUE_NAME = "Value";
+ private static final Configuration JSON_PATH_CONFIG =
+ Configuration.defaultConfiguration()
+ .addOptions(Option.SUPPRESS_EXCEPTIONS,
Option.DEFAULT_PATH_LEAF_TO_NULL);
+
private String string;
@Getter private FileObject file;
@@ -66,6 +78,14 @@ public class YamlReader {
private boolean useMap;
@Getter private Yaml yaml;
+ /** yaml field path */
+ private String[] fieldPaths;
+
+ private JsonPath[] compiledPaths;
+ private String[] discoveredFieldPaths;
+
+ private final Deque<Object[]> pendingRows = new ArrayDeque<>();
+
/** Bytes read from the last loaded file (for data volume tracking). */
@Getter private long bytesReadFromFile;
@@ -78,6 +98,26 @@ public class YamlReader {
this.yaml = new Yaml();
}
+ /**
+ * Sets the YAML field paths to be extracted and pre-compiles them into
{@link JsonPath} instances
+ * for better runtime performance.
+ *
+ * @param paths the array of field paths to extract; {@code null} clears all
configured paths
+ */
+ public void setFieldPaths(String[] paths) {
+ if (paths == null) {
+ this.fieldPaths = null;
+ this.compiledPaths = null;
+ return;
+ }
+
+ this.fieldPaths = paths.clone();
+ this.compiledPaths = new JsonPath[paths.length];
+ for (int i = 0; i < paths.length; i++) {
+ this.compiledPaths[i] =
JsonPath.compile(YamlPathResolver.normalize(paths[i]));
+ }
+ }
+
public void loadFile(FileObject file) throws Exception {
this.file = file;
@@ -107,6 +147,14 @@ public class YamlReader {
/** get row */
public Object[] getRow(IRowMeta rowMeta) {
+ if (!pendingRows.isEmpty()) {
+ Object[] row = pendingRows.poll();
+ if (pendingRows.isEmpty()) {
+ finishDocument();
+ }
+ return row;
+ }
+
if (document == null) {
getNextDocument();
}
@@ -122,19 +170,41 @@ public class YamlReader {
return row;
}
+ /**
+ * Ensures that the configured field paths have been compiled into {@link
JsonPath} instances.
+ *
+ * @param rowMeta the row metadata used to determine field paths when they
have not been
+ * explicitly configured
+ */
+ private void ensurePathsCompiled(IRowMeta rowMeta) {
+ if (compiledPaths != null && compiledPaths.length == rowMeta.size()) {
+ return;
+ }
+
+ String[] paths = new String[rowMeta.size()];
+ if (discoveredFieldPaths != null && discoveredFieldPaths.length ==
rowMeta.size()) {
+ System.arraycopy(discoveredFieldPaths, 0, paths, 0, paths.length);
+ } else {
+ for (int i = 0; i < rowMeta.size(); i++) {
+ paths[i] = rowMeta.getValueMeta(i).getName();
+ }
+ }
+ setFieldPaths(paths);
+ }
+
private Object[] fetchRow(IRowMeta rowMeta) {
+ ensurePathsCompiled(rowMeta);
+
if (isMapUsed()) {
- return processMapRow(rowMeta, (Map<?, ?>) document);
+ return processMapRow(rowMeta, document, false);
}
return processListRow(rowMeta);
}
private Object[] handleCurrentListValue(IRowMeta rowMeta) {
- List<?> list = (List<?>) document;
-
- if (list.size() == 1 && list.getFirst() instanceof Map<?, ?> map) {
- return processMapRow(rowMeta, map);
+ if (dataList instanceof Map<?, ?>) {
+ return processMapRow(rowMeta, dataList, true);
}
IValueMeta valueMeta = rowMeta.getValueMeta(0);
@@ -151,20 +221,92 @@ public class YamlReader {
return row;
}
- private Object[] processMapRow(IRowMeta rowMeta, Map<?, ?> map) {
- Object[] row = new Object[rowMeta.size()];
+ private Object[] processMapRow(IRowMeta rowMeta, Object root, boolean
listElement) {
+ if (pendingRows.isEmpty()) {
+ prepareRowsFromPaths(rowMeta, root, listElement);
+ }
- for (int i = 0; i < rowMeta.size(); i++) {
- IValueMeta valueMeta = rowMeta.getValueMeta(i);
+ Object[] row = pendingRows.poll();
+ if (pendingRows.isEmpty()) {
+ finishDocument();
+ }
+ return row == null ? new Object[0] : row;
+ }
- Object value =
- Utils.isEmpty(valueMeta.getName()) ? document.toString() :
map.get(valueMeta.getName());
+ private void prepareRowsFromPaths(IRowMeta rowMeta, Object root, boolean
listElement) {
+ List<List<Object>> columnValues = new ArrayList<>(compiledPaths.length);
+ int rowCount = 1;
- row[i] = getValue(value, valueMeta);
+ for (int pathIndex = 0; pathIndex < compiledPaths.length; pathIndex++) {
+ JsonPath path = pathForRoot(compiledPaths[pathIndex],
fieldPaths[pathIndex], listElement);
+ List<Object> values = readPathAsList(root, path);
+ if (values.size() > rowCount) {
+ rowCount = values.size();
+ }
+ columnValues.add(values);
}
- finishDocument();
- return row;
+ for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) {
+ Object[] row = new Object[rowMeta.size()];
+ for (int col = 0; col < rowMeta.size(); col++) {
+ List<Object> values = columnValues.get(col);
+ Object raw = pickValue(values, rowIndex, rowCount);
+ row[col] = getValue(raw, rowMeta.getValueMeta(col));
+ }
+ pendingRows.add(row);
+ }
+ }
+
+ private static JsonPath pathForRoot(
+ JsonPath compiledPath, String originalPath, boolean listElement) {
+ if (!listElement || originalPath == null || !originalPath.contains("[*]"))
{
+ return compiledPath;
+ }
+
+ String adapted = originalPath.replaceFirst("\\$\\[\\*]", "\\$");
+ return JsonPath.compile(YamlPathResolver.normalize(adapted));
+ }
+
+ private static Object pickValue(List<Object> values, int rowIndex, int
rowCount) {
+ if (values == null || values.isEmpty()) {
+ return null;
+ }
+
+ if (values.size() == 1 && rowCount > 1) {
+ return values.getFirst();
+ }
+ if (rowIndex < values.size()) {
+ return values.get(rowIndex);
+ }
+ return null;
+ }
+
+ private List<Object> readPathAsList(Object root, JsonPath path) {
+ if (root == null) {
+ return List.of((Object) null);
+ }
+
+ try {
+ ReadContext context = JsonPath.using(JSON_PATH_CONFIG).parse(root);
+ Object result = context.read(path);
+ return toValueList(result);
+ } catch (Exception e) {
+ return List.of((Object) null);
+ }
+ }
+
+ private static List<Object> toValueList(Object result) {
+ if (result == null) {
+ return List.of((Object) null);
+ }
+
+ if (result instanceof List<?> list) {
+ if (list.isEmpty()) {
+ return List.of((Object) null);
+ }
+ return new ArrayList<>(list);
+ }
+ return List.of(result);
}
private Object[] advanceListIterator() {
@@ -178,6 +320,7 @@ public class YamlReader {
}
private void getNextDocument() {
+ pendingRows.clear();
// See if we have another document
if (this.documentIt.hasNext()) {
// We have another document
@@ -237,6 +380,7 @@ public class YamlReader {
private void finishDocument() {
this.document = null;
+ pendingRows.clear();
}
private boolean isFinishedDocument() {
@@ -345,39 +489,80 @@ public class YamlReader {
/** get fields. */
public RowMeta getFields() {
RowMeta rowMeta = new RowMeta();
-
- for (Object data : documents) {
- if (data instanceof Map<?, ?> map) {
- appendFromMap(rowMeta, map);
- } else if (data instanceof List<?> list) {
- appendFromList(rowMeta, list);
+ List<String> paths = getDiscoveredPaths();
+ discoveredFieldPaths = paths.toArray(new String[0]);
+
+ for (String path : paths) {
+ Object sample = null;
+ for (Object data : documents) {
+ sample = readSampleValue(data, path);
+ if (sample != null) {
+ break;
+ }
}
+ addField(rowMeta, path, sample);
}
return rowMeta;
}
- private void appendFromList(RowMeta rowMeta, List<?> list) {
- if (list.isEmpty()) {
- return;
+ /** Returns JsonPath expressions discovered in the loaded YAML documents. */
+ public List<String> getDiscoveredPaths() {
+ Set<String> paths = new LinkedHashSet<>();
+ for (Object data : documents) {
+ collectPaths(data, YamlPathResolver.DOLLAR, paths);
}
+ return new ArrayList<>(paths);
+ }
- Object first = list.getFirst();
- if (list.size() == 1 && first instanceof Map<?, ?> map) {
- appendFromMap(rowMeta, map);
- } else {
- addField(rowMeta, DEFAULT_LIST_VALUE_NAME, first);
+ private Object readSampleValue(Object root, String path) {
+ if (root == null) {
+ return null;
}
+ List<Object> values = readPathAsList(root, JsonPath.compile(path));
+ return values.isEmpty() ? null : values.getFirst();
}
- private void appendFromMap(RowMeta rowMeta, Map<?, ?> map) {
- for (Map.Entry<?, ?> entry : map.entrySet()) {
- addField(rowMeta, entry.getKey(), entry.getValue());
+ private void collectPaths(Object node, String prefix, Set<String> paths) {
+ if (node instanceof Map<?, ?> map) {
+ for (Map.Entry<?, ?> entry : map.entrySet()) {
+ String key = String.valueOf(entry.getKey());
+ String segment = formatPathSegment(key);
+ String childPath =
+ YamlPathResolver.DOLLAR.equals(prefix)
+ ? prefix + "." + segment
+ : prefix + "." + segment;
+ Object value = entry.getValue();
+ if (value instanceof Map || value instanceof List) {
+ collectPaths(value, childPath, paths);
+ } else {
+ paths.add(childPath);
+ }
+ }
+ } else if (node instanceof List<?> list) {
+ if (!list.isEmpty()) {
+ Object first = list.getFirst();
+ if (first instanceof Map || first instanceof List) {
+ collectPaths(first, prefix + "[*]", paths);
+ } else {
+ paths.add(prefix + "[*]");
+ }
+ }
+ } else if (YamlPathResolver.DOLLAR.equals(prefix)) {
+ paths.add(YamlPathResolver.DOLLAR);
}
}
- private void addField(RowMeta rowMeta, Object key, Object value) {
- String fieldName = String.valueOf(key);
+ private static String formatPathSegment(String key) {
+ if (key.contains(".") || key.contains(" ") || key.contains("[")) {
+ return "['" + key.replace("'", "\\'") + "']";
+ }
+
+ return key;
+ }
+
+ private void addField(RowMeta rowMeta, String path, Object value) {
+ String fieldName = fieldNameFromPath(path);
IValueMeta valueMeta;
try {
@@ -388,4 +573,33 @@ public class YamlReader {
rowMeta.addValueMeta(valueMeta);
}
+
+ static String fieldNameFromPath(String path) {
+ if (Utils.isEmpty(path) || YamlPathResolver.DOLLAR.equals(path)) {
+ return path;
+ }
+ if ("$[*]".equals(path)) {
+ return DEFAULT_LIST_VALUE_NAME;
+ }
+
+ int bracket = path.lastIndexOf("['");
+ if (bracket >= 0) {
+ int quoteEnd = path.indexOf("']", bracket);
+ if (quoteEnd > bracket) {
+ return path.substring(bracket + 2, quoteEnd).replace("\\'", "'");
+ }
+ }
+
+ int dot = path.lastIndexOf('.');
+ if (dot >= 0 && dot < path.length() - 1) {
+ String leaf = path.substring(dot + 1);
+ int open = leaf.indexOf('[');
+ return open >= 0 ? leaf.substring(0, open) : leaf;
+ }
+
+ if (path.startsWith(YamlPathResolver.DOLLAR)) {
+ return path.substring(1);
+ }
+ return path;
+ }
}
diff --git
a/plugins/transforms/yamlinput/src/main/resources/org/apache/hop/pipeline/transforms/yamlinput/messages/messages_en_US.properties
b/plugins/transforms/yamlinput/src/main/resources/org/apache/hop/pipeline/transforms/yamlinput/messages/messages_en_US.properties
index cda9aa50ba..9da646c273 100644
---
a/plugins/transforms/yamlinput/src/main/resources/org/apache/hop/pipeline/transforms/yamlinput/messages/messages_en_US.properties
+++
b/plugins/transforms/yamlinput/src/main/resources/org/apache/hop/pipeline/transforms/yamlinput/messages/messages_en_US.properties
@@ -16,7 +16,7 @@
# limitations under the License.
#
-YamlInput.Description=Read YAML source (file or stream) parse them and convert
them to rows and writes these to one or more output.
+YamlInput.Description=Read YAML source (file or stream), extract fields with
JsonPath expressions, and output rows.
YamlInput.Error.FileSizeZero=File [{0}] is empty\!
YamlInput.ErrorInTransformRunning=Error reading from Yaml source\! {0}
YamlInput.Exception.CouldnotFindField=Could not find field ''{0}'' in row\!
@@ -61,8 +61,8 @@ YamlInputDialog.FieldsTable.Name.Column.Tooltip=Field name in
the returned strea
YamlInputDialog.FieldsTable.Precision.Column=Precision
YamlInputDialog.FieldsTable.TrimType.Column=Trim type
YamlInputDialog.FieldsTable.Type.Column=Type
-YamlInputDialog.FieldsTable.XPath.Column=Key
-YamlInputDialog.FieldsTable.XPath.Column.Tooltip=Path to extract from the files
+YamlInputDialog.FieldsTable.XPath.Column=Path
+YamlInputDialog.FieldsTable.XPath.Column.Tooltip=JsonPath expression to
extract from the YAML (e.g. $.table.name or $.tables[0].name). Legacy flat keys
such as key1 are treated as $.key1.
YamlInputDialog.File.Tab=File
YamlInputDialog.Filename.Label=File or directory
YamlInputDialog.FilenameAdd.Button=&Add
diff --git
a/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputTests.java
b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputTests.java
index d996d88e20..8f4723f88f 100644
---
a/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputTests.java
+++
b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputTests.java
@@ -81,6 +81,7 @@ class YamlInputTests {
@Test
void testInit_shouldBuildRowMeta() {
YamlInputField field = new YamlInputField();
+ field.setName("user_name");
field.setPath("user.name");
field.setType(IValueMeta.TYPE_STRING);
field.setTrimType(YamlInputField.TYPE_TRIM_BOTH);
@@ -98,6 +99,8 @@ class YamlInputTests {
assertNotNull(input.getData().rowMeta);
assertEquals(1, input.getData().nrInputFields);
+ assertEquals("user_name",
input.getData().rowMeta.getValueMeta(0).getName());
+ assertEquals("$.user.name", input.getData().fieldPaths[0]);
}
@Test
diff --git
a/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlPathResolverTests.java
b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlPathResolverTests.java
new file mode 100644
index 0000000000..92d43a2c10
--- /dev/null
+++
b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlPathResolverTests.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.yamlinput;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.junit.rules.RestoreHopEnvironmentExtension;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(RestoreHopEnvironmentExtension.class)
+class YamlPathResolverTests {
+
+ @BeforeAll
+ static void init() throws HopException {
+ HopEnvironment.init();
+ }
+
+ @Test
+ void normalize_legacyFlatKey() {
+ assertEquals("$.key1", YamlPathResolver.normalize("key1"));
+ }
+
+ @Test
+ void normalize_dottedPath() {
+ assertEquals("$.table.name", YamlPathResolver.normalize("table.name"));
+ }
+
+ @Test
+ void normalize_jsonPathUnchanged() {
+ assertEquals("$.table.name", YamlPathResolver.normalize("$.table.name"));
+ assertEquals("$.tables[0].name",
YamlPathResolver.normalize("$.tables[0].name"));
+ assertEquals("$[*].name", YamlPathResolver.normalize("$[*].name"));
+ }
+
+ @Test
+ void normalize_emptyPath() {
+ assertEquals(YamlPathResolver.DOLLAR, YamlPathResolver.normalize(""));
+ assertEquals(YamlPathResolver.DOLLAR, YamlPathResolver.normalize(null));
+ }
+
+ @Test
+ void resolvePath_substitutesVariablesWithoutManglingWildcards() {
+ Variables variables = new Variables();
+ variables.setVariable("TABLE", "table");
+
+ assertEquals(
+ "$.table.name",
+ YamlPathResolver.resolvePath(variables, "$." +
variables.getVariable("TABLE") + ".name"));
+ assertEquals("$.items[*].id", YamlPathResolver.resolvePath(variables,
"$.items[*].id"));
+ }
+}
diff --git
a/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReaderTests.java
b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReaderTests.java
index fed1ede508..08e3050403 100644
---
a/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReaderTests.java
+++
b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReaderTests.java
@@ -25,11 +25,14 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.FileWriter;
import java.nio.file.Path;
import java.util.Date;
+import java.util.List;
import org.apache.commons.vfs2.FileObject;
import org.apache.hop.core.HopEnvironment;
import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopPluginException;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaFactory;
import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.junit.rules.RestoreHopEnvironmentExtension;
import org.junit.jupiter.api.BeforeAll;
@@ -156,7 +159,7 @@ class YamlReaderTests {
}
@Test
- void testNestedMapToString() {
+ void testNestedMapToString() throws HopPluginException {
String yaml = """
person:
name: Tom
@@ -165,7 +168,10 @@ class YamlReaderTests {
reader.loadString(yaml);
- RowMeta rowMeta = reader.getFields();
+ RowMeta rowMeta = new RowMeta();
+ reader.setFieldPaths(new String[] {"person"});
+ rowMeta.addValueMeta(ValueMetaFactory.createValueMeta("person",
IValueMeta.TYPE_STRING));
+
Object[] row = reader.getRow(rowMeta);
assertNotNull(row[0]);
@@ -173,6 +179,144 @@ class YamlReaderTests {
assertTrue(row[0].toString().contains("age"));
}
+ @Test
+ void testJsonPath_nestedObjectFields() throws Exception {
+ String yaml = """
+ table:
+ name: t_user
+ startId: 1000
+ """;
+
+ reader.loadString(yaml);
+
+ RowMeta rowMeta =
+ buildRowMeta(
+ new String[] {"table_name", "start_id"},
+ new String[] {"$.table.name", "$.table.startId"},
+ new int[] {IValueMeta.TYPE_STRING, IValueMeta.TYPE_INTEGER});
+
+ Object[] row = reader.getRow(rowMeta);
+
+ assertEquals("t_user", row[0]);
+ assertEquals(1000L, row[1]);
+ }
+
+ @Test
+ void testJsonPath_legacyFlatKeyBackwardCompatible() throws Exception {
+ String yaml = """
+ key1: alpha
+ key2: 42
+ """;
+
+ reader.loadString(yaml);
+
+ RowMeta rowMeta =
+ buildRowMeta(
+ new String[] {"field1", "field2"},
+ new String[] {"key1", "key2"},
+ new int[] {IValueMeta.TYPE_STRING, IValueMeta.TYPE_INTEGER});
+
+ Object[] row = reader.getRow(rowMeta);
+
+ assertEquals("alpha", row[0]);
+ assertEquals(42L, row[1]);
+ }
+
+ @Test
+ void testJsonPath_arrayIndex() throws Exception {
+ String yaml = """
+ tables:
+ - name: t_user
+ - name: t_order
+ """;
+
+ reader.loadString(yaml);
+
+ RowMeta rowMeta =
+ buildRowMeta(
+ new String[] {"first_table", "second_table"},
+ new String[] {"$.tables[0].name", "$.tables[1].name"},
+ new int[] {IValueMeta.TYPE_STRING, IValueMeta.TYPE_STRING});
+
+ Object[] row = reader.getRow(rowMeta);
+
+ assertEquals("t_user", row[0]);
+ assertEquals("t_order", row[1]);
+ }
+
+ @Test
+ void testJsonPath_wildcardProducesMultipleRows() throws Exception {
+ String yaml = """
+ tables:
+ - name: t_user
+ - name: t_order
+ """;
+
+ reader.loadString(yaml);
+
+ RowMeta rowMeta =
+ buildRowMeta(
+ new String[] {"name"},
+ new String[] {"$.tables[*].name"},
+ new int[] {IValueMeta.TYPE_STRING});
+
+ Object[] row1 = reader.getRow(rowMeta);
+ Object[] row2 = reader.getRow(rowMeta);
+
+ assertEquals("t_user", row1[0]);
+ assertEquals("t_order", row2[0]);
+ }
+
+ @Test
+ void testGetFieldsAndGetRow_listOfMap() {
+ String yaml = """
+ - name: Alice
+ age: 25
+ """;
+
+ reader.loadString(yaml);
+
+ RowMeta rowMeta = reader.getFields();
+ assertEquals(List.of("$[*].name", "$[*].age"),
reader.getDiscoveredPaths());
+
+ Object[] row = reader.getRow(rowMeta);
+
+ assertEquals("Alice", row[0]);
+ assertEquals(25L, row[1]);
+ }
+
+ @Test
+ void testGetDiscoveredPaths_nestedStructure() {
+ String yaml = """
+ table:
+ name: t_user
+ startId: 1000
+ """;
+
+ reader.loadString(yaml);
+
+ assertEquals(List.of("$.table.name", "$.table.startId"),
reader.getDiscoveredPaths());
+ }
+
+ @Test
+ void testFieldNameFromPath() {
+ assertEquals("name", YamlReader.fieldNameFromPath("$.table.name"));
+ assertEquals("name", YamlReader.fieldNameFromPath("$.tables[0].name"));
+ assertEquals("name", YamlReader.fieldNameFromPath("$.tables[*].name"));
+ assertEquals("name", YamlReader.fieldNameFromPath("$[*].name"));
+ assertEquals("Value", YamlReader.fieldNameFromPath("$[*]"));
+ assertEquals("my field", YamlReader.fieldNameFromPath("$['my
field'].value"));
+ }
+
+ private RowMeta buildRowMeta(String[] names, String[] paths, int[] types)
throws Exception {
+ reader.setFieldPaths(paths);
+ RowMeta rowMeta = new RowMeta();
+ for (int i = 0; i < names.length; i++) {
+ rowMeta.addValueMeta(ValueMetaFactory.createValueMeta(names[i],
types[i]));
+ }
+ return rowMeta;
+ }
+
@Test
void testEmptyYaml() {
reader.loadString("");
@@ -187,15 +331,15 @@ class YamlReaderTests {
void testGetFields_TypeInference() {
String yaml =
"""
- - intField: 1
- longField: 2
- doubleField: 1.5
- boolField: true
- bigIntField: 99999999999999999999
- bigDecField: 9999999999999999999999999.55
- byteField: 1
- dateField: 2026-03-04T12:03:01
- """;
+ - intField: 1
+ longField: 2
+ doubleField: 1.5
+ boolField: true
+ bigIntField:
99999999999999999999
+ bigDecField:
9999999999999999999999999.55
+ byteField: 1
+ dateField: 2026-03-04T12:03:01
+ """;
reader.loadString(yaml);
