This is an automated email from the ASF dual-hosted git repository.

bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/main by this push:
     new d0709483b9 YAML input: quoted dates and booleans get the type their 
field declar… (#9519)
d0709483b9 is described below

commit d0709483b91d9ac065d32417b980193a380d5d02
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Tue Nov 17 09:52:23 2026 +0100

    YAML input: quoted dates and booleans get the type their field declar… 
(#9519)
    
    * YAML input: quoted dates and booleans get the type their field declares, 
fixes #9510
    
    * fix remarks
---
 .../transforms/0088-yaml-input-validate.hpl        | 24 +++++++++++++
 .../transforms/datasets/golden-yaml-input.csv      | 12 +++----
 integration-tests/transforms/files/yaml-input.yaml |  2 ++
 .../metadata/dataset/golden-yaml-input.json        | 16 +++++++++
 .../unit-test/0088-yaml-input-validate UNIT.json   |  8 +++++
 .../pipeline/transforms/yamlinput/YamlInput.java   |  7 ++++
 .../pipeline/transforms/yamlinput/YamlReader.java  | 39 +++++++++++++++++++++-
 .../transforms/yamlinput/YamlReaderTests.java      | 26 +++++++++++++++
 8 files changed, 127 insertions(+), 7 deletions(-)

diff --git a/integration-tests/transforms/0088-yaml-input-validate.hpl 
b/integration-tests/transforms/0088-yaml-input-validate.hpl
index 8ccc00fff5..c4813695b2 100644
--- a/integration-tests/transforms/0088-yaml-input-validate.hpl
+++ b/integration-tests/transforms/0088-yaml-input-validate.hpl
@@ -119,6 +119,30 @@ limitations under the License.
         <decimal/>
         <group/>
       </field>
+      <field>
+        <name>release-date</name>
+        <path>release-date</path>
+        <type>Date</type>
+        <length>-1</length>
+        <format>yyyy-MM-dd</format>
+        <trim_type>none</trim_type>
+        <precision>-1</precision>
+        <currency/>
+        <decimal/>
+        <group/>
+      </field>
+      <field>
+        <name>incubating</name>
+        <path>incubating</path>
+        <type>Boolean</type>
+        <length>-1</length>
+        <format/>
+        <trim_type>none</trim_type>
+        <precision>-1</precision>
+        <currency/>
+        <decimal/>
+        <group/>
+      </field>
       <field>
         <name>used-apache-projects</name>
         <path>used-apache-projects</path>
diff --git a/integration-tests/transforms/datasets/golden-yaml-input.csv 
b/integration-tests/transforms/datasets/golden-yaml-input.csv
index 84137e344f..8d8c0fc5ec 100644
--- a/integration-tests/transforms/datasets/golden-yaml-input.csv
+++ b/integration-tests/transforms/datasets/golden-yaml-input.csv
@@ -1,6 +1,6 @@
-organisation,acronym,project,javaFileCount,awesome,used-apache-projects,statistics
-The Apache Software Foundation,The ASF,Apache 
Hop,4447,Y,Spark,"{number-of-libs:  1159,number-of-messages-bundles:  
13025,integration-test-projects:  {count:  39,pipelines:  1032,workflows:  
498}}"
-The Apache Software Foundation,The ASF,Apache 
Hop,4447,Y,Flink,"{number-of-libs:  1159,number-of-messages-bundles:  
13025,integration-test-projects:  {count:  39,pipelines:  1032,workflows:  
498}}"
-The Apache Software Foundation,The ASF,Apache 
Hop,4447,Y,Beam,"{number-of-libs:  1159,number-of-messages-bundles:  
13025,integration-test-projects:  {count:  39,pipelines:  1032,workflows:  
498}}"
-The Apache Software Foundation,The ASF,Apache 
Hop,4447,Y,Commons,"{number-of-libs:  1159,number-of-messages-bundles:  
13025,integration-test-projects:  {count:  39,pipelines:  1032,workflows:  
498}}"
-The Apache Software Foundation,The ASF,Apache Hop,4447,Y,VFS,"{number-of-libs: 
 1159,number-of-messages-bundles:  13025,integration-test-projects:  {count:  
39,pipelines:  1032,workflows:  498}}"
+organisation,acronym,project,javaFileCount,awesome,used-apache-projects,statistics,release-date,incubating
+The Apache Software Foundation,The ASF,Apache 
Hop,4447,Y,Spark,"{number-of-libs:  1159,number-of-messages-bundles:  
13025,integration-test-projects:  {count:  39,pipelines:  1032,workflows:  
498}}",2020-02-28,N
+The Apache Software Foundation,The ASF,Apache 
Hop,4447,Y,Flink,"{number-of-libs:  1159,number-of-messages-bundles:  
13025,integration-test-projects:  {count:  39,pipelines:  1032,workflows:  
498}}",2020-02-28,N
+The Apache Software Foundation,The ASF,Apache 
Hop,4447,Y,Beam,"{number-of-libs:  1159,number-of-messages-bundles:  
13025,integration-test-projects:  {count:  39,pipelines:  1032,workflows:  
498}}",2020-02-28,N
+The Apache Software Foundation,The ASF,Apache 
Hop,4447,Y,Commons,"{number-of-libs:  1159,number-of-messages-bundles:  
13025,integration-test-projects:  {count:  39,pipelines:  1032,workflows:  
498}}",2020-02-28,N
+The Apache Software Foundation,The ASF,Apache Hop,4447,Y,VFS,"{number-of-libs: 
 1159,number-of-messages-bundles:  13025,integration-test-projects:  {count:  
39,pipelines:  1032,workflows:  498}}",2020-02-28,N
diff --git a/integration-tests/transforms/files/yaml-input.yaml 
b/integration-tests/transforms/files/yaml-input.yaml
index 6b9bea2b92..6616f16f03 100644
--- a/integration-tests/transforms/files/yaml-input.yaml
+++ b/integration-tests/transforms/files/yaml-input.yaml
@@ -23,6 +23,8 @@
   project: "Apache Hop"
   javaFileCount: 4447
   awesome: true
+  release-date: '2020-02-28'
+  incubating: 'N'
   used-apache-projects:
     - Spark
     - Flink
diff --git 
a/integration-tests/transforms/metadata/dataset/golden-yaml-input.json 
b/integration-tests/transforms/metadata/dataset/golden-yaml-input.json
index a0c797199b..1d5dd2f3f0 100644
--- a/integration-tests/transforms/metadata/dataset/golden-yaml-input.json
+++ b/integration-tests/transforms/metadata/dataset/golden-yaml-input.json
@@ -58,6 +58,22 @@
       "field_precision": -1,
       "field_name": "statistics",
       "field_format": ""
+    },
+    {
+      "field_comment": "",
+      "field_length": -1,
+      "field_type": 3,
+      "field_precision": -1,
+      "field_name": "release-date",
+      "field_format": "yyyy-MM-dd"
+    },
+    {
+      "field_comment": "",
+      "field_length": -1,
+      "field_type": 4,
+      "field_precision": -1,
+      "field_name": "incubating",
+      "field_format": ""
     }
   ],
   "folder_name": ""
diff --git 
a/integration-tests/transforms/metadata/unit-test/0088-yaml-input-validate 
UNIT.json 
b/integration-tests/transforms/metadata/unit-test/0088-yaml-input-validate 
UNIT.json
index 9d4c6b1262..622105fb7d 100644
--- a/integration-tests/transforms/metadata/unit-test/0088-yaml-input-validate 
UNIT.json        
+++ b/integration-tests/transforms/metadata/unit-test/0088-yaml-input-validate 
UNIT.json        
@@ -36,6 +36,14 @@
         {
           "transform_field": "statistics",
           "data_set_field": "statistics"
+        },
+        {
+          "transform_field": "release-date",
+          "data_set_field": "release-date"
+        },
+        {
+          "transform_field": "incubating",
+          "data_set_field": "incubating"
         }
       ],
       "field_order": [
diff --git 
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java
 
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java
index df807ed196..27baffdefa 100644
--- 
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java
+++ 
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java
@@ -430,6 +430,13 @@ public class YamlInput extends 
BaseTransform<YamlInputMeta, YamlInputData> {
         try {
           IValueMeta valueMeta = ValueMetaFactory.createValueMeta(fieldName, 
field.getType());
           valueMeta.setTrimType(field.getTrimType());
+          // The field's format, as getFields() declares it: text in the YAML 
is converted with it
+          valueMeta.setLength(field.getLength());
+          valueMeta.setPrecision(field.getPrecision());
+          valueMeta.setConversionMask(field.getFormat());
+          valueMeta.setDecimalSymbol(field.getDecimalSymbol());
+          valueMeta.setGroupingSymbol(field.getGroupSymbol());
+          valueMeta.setCurrencySymbol(field.getCurrencySymbol());
           data.rowMeta.addValueMeta(valueMeta);
         } catch (Exception e) {
           logError("Unable to create value meta", e);
diff --git 
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReader.java
 
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReader.java
index a531445867..79f9cbf64a 100644
--- 
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReader.java
+++ 
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReader.java
@@ -38,12 +38,14 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.hop.core.Const;
 import org.apache.hop.core.exception.HopPluginException;
+import org.apache.hop.core.exception.HopValueException;
 import org.apache.hop.core.io.CountingInputStream;
 import org.apache.hop.core.row.IRowMeta;
 import org.apache.hop.core.row.IValueMeta;
 import org.apache.hop.core.row.RowMeta;
 import org.apache.hop.core.row.value.ValueMetaFactory;
 import org.apache.hop.core.row.value.ValueMetaNone;
+import org.apache.hop.core.row.value.ValueMetaString;
 import org.apache.hop.core.util.Utils;
 import org.apache.hop.core.vfs.HopVfs;
 import org.yaml.snakeyaml.Yaml;
@@ -437,7 +439,9 @@ public class YamlReader {
       case IValueMeta.TYPE_INTEGER -> convertToInteger(normalizedValue);
       case IValueMeta.TYPE_NUMBER -> convertToDouble(normalizedValue);
       case IValueMeta.TYPE_BIGNUMBER -> convertToBigDecimal(normalizedValue);
-      case IValueMeta.TYPE_BOOLEAN, IValueMeta.TYPE_DATE, 
IValueMeta.TYPE_BINARY -> normalizedValue;
+      case IValueMeta.TYPE_DATE -> convertToDate(normalizedValue, valueMeta);
+      case IValueMeta.TYPE_BOOLEAN -> convertToBoolean(normalizedValue, 
valueMeta);
+      case IValueMeta.TYPE_BINARY -> normalizedValue;
       default -> convertToString(normalizedValue, valueMeta);
     };
   }
@@ -467,6 +471,39 @@ public class YamlReader {
     return s.substring(0, end);
   }
 
+  /**
+   * object -> date. SnakeYAML reads an unquoted ISO date as a {@link 
java.util.Date}; a quoted one,
+   * or another format, stays text and is parsed with the field's format, so 
the value matches the
+   * Date type the field declares.
+   */
+  private Object convertToDate(Object value, IValueMeta valueMeta) {
+    if (value instanceof java.util.Date) {
+      return value;
+    }
+    return convertText(value, valueMeta);
+  }
+
+  /** object -> boolean: YAML booleans stay, text (Y/N, true/false) is 
converted. */
+  private Object convertToBoolean(Object value, IValueMeta valueMeta) {
+    if (value instanceof Boolean) {
+      return value;
+    }
+    return convertText(value, valueMeta);
+  }
+
+  /** Text to the field's type with its format, as Hop converts a String 
field. */
+  private static Object convertText(Object value, IValueMeta valueMeta) {
+    ValueMetaString text = new ValueMetaString(valueMeta.getName());
+    text.setConversionMask(valueMeta.getConversionMask());
+    text.setDateFormatLenient(valueMeta.isDateFormatLenient());
+    try {
+      return valueMeta.convertData(text, value.toString());
+    } catch (HopValueException e) {
+      throw new IllegalArgumentException(
+          "Unable to convert '" + value + "' of field '" + valueMeta.getName() 
+ "'", e);
+    }
+  }
+
   /** object -> long */
   @SuppressWarnings("javabugs:S2259") // the value is never null when this is 
reached
   private Long convertToInteger(Object value) {
diff --git 
a/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReaderTests.java
 
b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReaderTests.java
index b3bbebdce5..02638392e0 100644
--- 
a/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReaderTests.java
+++ 
b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReaderTests.java
@@ -33,7 +33,10 @@ import org.apache.hop.core.exception.HopException;
 import org.apache.hop.core.exception.HopPluginException;
 import org.apache.hop.core.row.IValueMeta;
 import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaBoolean;
+import org.apache.hop.core.row.value.ValueMetaDate;
 import org.apache.hop.core.row.value.ValueMetaFactory;
+import org.apache.hop.core.row.value.ValueMetaInteger;
 import org.apache.hop.core.vfs.HopVfs;
 import org.apache.hop.junit.rules.RestoreHopEnvironmentExtension;
 import org.junit.jupiter.api.BeforeAll;
@@ -102,6 +105,29 @@ class YamlReaderTests {
     assertInstanceOf(Date.class, row[2]);
   }
 
+  @Test
+  void aQuotedDateBecomesTheDateItsFieldDeclares() {
+    String yaml = """
+                               order_id: 1
+                               order_date: '2020-02-28'
+                               shipped: 'Y'
+                               """;
+
+    reader.loadString(yaml);
+
+    RowMeta rowMeta = new RowMeta();
+    rowMeta.addValueMeta(new ValueMetaInteger("order_id"));
+    ValueMetaDate date = new ValueMetaDate("order_date");
+    date.setConversionMask("yyyy-MM-dd");
+    rowMeta.addValueMeta(date);
+    rowMeta.addValueMeta(new ValueMetaBoolean("shipped"));
+    Object[] row = reader.getRow(rowMeta);
+
+    assertEquals(1L, row[0]);
+    assertInstanceOf(Date.class, row[1]);
+    assertEquals(true, row[2]);
+  }
+
   @Test
   void testLoadString_ListMultipleValues() {
     String yaml = """

Reply via email to