This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new b82cbee8fa Fix #6144 the outputValue initialization issue in the
Formula transform (#6158)
b82cbee8fa is described below
commit b82cbee8fa2a324b98ed1c0128a110e205d4654e
Author: lance <[email protected]>
AuthorDate: Tue Dec 23 21:48:26 2025 +0800
Fix #6144 the outputValue initialization issue in the Formula transform
(#6158)
* Fix the outputValue initialization issue in the Formula transform
Signed-off-by: lance <[email protected]>
* add formula error type,integration-tests
Signed-off-by: lance <[email protected]>
* Add test to workflow
---------
Signed-off-by: lance <[email protected]>
Co-authored-by: Hans Van Akelyen <[email protected]>
---
.../transforms/0042-formula-logical-error.hpl | 211 +++++++++++++++++++++
.../datasets/golden-formula-logical-error.csv | 3 +
.../transforms/main-0042-formula-logical.hwf | 14 +-
.../dataset/golden-formula-logical-error.json | 48 +++++
.../unit-test/0042-formula-logical-error UNIT.json | 48 +++++
.../hop/pipeline/transforms/formula/Formula.java | 4 +-
.../pipeline/transforms/formula/FormulaTests.java | 95 ++++++++++
7 files changed, 415 insertions(+), 8 deletions(-)
diff --git a/integration-tests/transforms/0042-formula-logical-error.hpl
b/integration-tests/transforms/0042-formula-logical-error.hpl
new file mode 100644
index 0000000000..74e9168323
--- /dev/null
+++ b/integration-tests/transforms/0042-formula-logical-error.hpl
@@ -0,0 +1,211 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0042-formula-logical-error</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2025/12/04 00:09:25.633</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2025/12/04 00:09:25.633</modified_date>
+ </info>
+ <notepads>
+ <notepad>
+ <backgroundcolorblue>251</backgroundcolorblue>
+ <backgroundcolorgreen>232</backgroundcolorgreen>
+ <backgroundcolorred>201</backgroundcolorred>
+ <bordercolorblue>90</bordercolorblue>
+ <bordercolorgreen>58</bordercolorgreen>
+ <bordercolorred>14</bordercolorred>
+ <fontbold>N</fontbold>
+ <fontcolorblue>90</fontcolorblue>
+ <fontcolorgreen>58</fontcolorgreen>
+ <fontcolorred>14</fontcolorred>
+ <fontitalic>N</fontitalic>
+ <fontname>Microsoft YaHei UI</fontname>
+ <fontsize>9</fontsize>
+ <height>24</height>
+ <xloc>64</xloc>
+ <yloc>256</yloc>
+ <note>The formula IF([card] > 0, [card], NA()) fails because NA() is not
convertible to an integer type.</note>
+ <width>503</width>
+ </notepad>
+ </notepads>
+ <order>
+ <hop>
+ <from>card</from>
+ <to>privacy and card and size</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>privacy and card and size</from>
+ <to>verify</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>card</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>N</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>29</item>
+ <item/>
+ <item/>
+ </line>
+ <line>
+ <item>0</item>
+ <item>J</item>
+ <item>133181722624</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <currency/>
+ <set_empty_string>N</set_empty_string>
+ <name>card</name>
+ <format/>
+ <group/>
+ <decimal/>
+ <type>Integer</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <currency/>
+ <set_empty_string>N</set_empty_string>
+ <name>privacy</name>
+ <format/>
+ <group/>
+ <decimal/>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <currency/>
+ <set_empty_string>N</set_empty_string>
+ <name>size</name>
+ <format/>
+ <group/>
+ <decimal/>
+ <type>Integer</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>160</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>privacy and card and size</name>
+ <type>Formula</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <formulas>
+ <formula>
+ <field_name>privacy</field_name>
+ <formula>IF(isNA([privacy]), "N", [privacy])</formula>
+ <replace_field>privacy</replace_field>
+ <set_na>N</set_na>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ <value_type>2</value_type>
+ </formula>
+ <formula>
+ <field_name>card</field_name>
+ <formula>IF([card] > 0, [card], NA())</formula>
+ <replace_field>card</replace_field>
+ <set_na>N</set_na>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ <value_type>5</value_type>
+ </formula>
+ <formula>
+ <field_name>size_m</field_name>
+ <formula>([size]/1024/1024)</formula>
+ <replace_field/>
+ <set_na>N</set_na>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ <value_type>5</value_type>
+ </formula>
+ <formula>
+ <field_name>size_g</field_name>
+ <formula>([size]/1024/1024/1024)</formula>
+ <replace_field/>
+ <set_na>N</set_na>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ <value_type>5</value_type>
+ </formula>
+ </formulas>
+ <attributes/>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>160</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>verify</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>400</xloc>
+ <yloc>160</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-formula-logical-error.csv
b/integration-tests/transforms/datasets/golden-formula-logical-error.csv
new file mode 100644
index 0000000000..ffc952b7eb
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-formula-logical-error.csv
@@ -0,0 +1,3 @@
+card,privacy,size,size_m,size_g
+29,,,0,0
+,J,133181722624,127012,124
diff --git a/integration-tests/transforms/main-0042-formula-logical.hwf
b/integration-tests/transforms/main-0042-formula-logical.hwf
index 7b9b47cd1f..5779e72fb5 100644
--- a/integration-tests/transforms/main-0042-formula-logical.hwf
+++ b/integration-tests/transforms/main-0042-formula-logical.hwf
@@ -35,14 +35,15 @@ limitations under the License.
<description/>
<type>SPECIAL</type>
<attributes/>
- <repeat>N</repeat>
- <schedulerType>0</schedulerType>
- <intervalSeconds>0</intervalSeconds>
- <intervalMinutes>60</intervalMinutes>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
<hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
<minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
<weekDay>1</weekDay>
- <DayOfMonth>1</DayOfMonth>
<parallel>N</parallel>
<xloc>50</xloc>
<yloc>50</yloc>
@@ -57,6 +58,9 @@ limitations under the License.
<test_name>
<name>0042-formula-logical UNIT</name>
</test_name>
+ <test_name>
+ <name>0042-formula-logical-error UNIT</name>
+ </test_name>
</test_names>
<parallel>N</parallel>
<xloc>192</xloc>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-formula-logical-error.json
b/integration-tests/transforms/metadata/dataset/golden-formula-logical-error.json
new file mode 100644
index 0000000000..5375bbe713
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-formula-logical-error.json
@@ -0,0 +1,48 @@
+{
+ "base_filename": "golden-formula-logical-error.csv",
+ "name": "golden-formula-logical-error",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "card",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "privacy",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "size",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "size_m",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "size_g",
+ "field_format": "####0;-####0"
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/unit-test/0042-formula-logical-error
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0042-formula-logical-error
UNIT.json
new file mode 100644
index 0000000000..9db3617d0f
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0042-formula-logical-error
UNIT.json
@@ -0,0 +1,48 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "card",
+ "data_set_field": "card"
+ },
+ {
+ "transform_field": "privacy",
+ "data_set_field": "privacy"
+ },
+ {
+ "transform_field": "size",
+ "data_set_field": "size"
+ },
+ {
+ "transform_field": "size_m",
+ "data_set_field": "size_m"
+ },
+ {
+ "transform_field": "size_g",
+ "data_set_field": "size_g"
+ }
+ ],
+ "field_order": [
+ "card",
+ "privacy",
+ "size",
+ "size_m",
+ "size_g"
+ ],
+ "data_set_name": "golden-formula-logical-error",
+ "transform_name": "verify"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0042-formula-logical-error UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0042-formula-logical-error.hpl"
+}
\ No newline at end of file
diff --git
a/plugins/transforms/formula/src/main/java/org/apache/hop/pipeline/transforms/formula/Formula.java
b/plugins/transforms/formula/src/main/java/org/apache/hop/pipeline/transforms/formula/Formula.java
index d1eb7454c7..7318ff657a 100644
---
a/plugins/transforms/formula/src/main/java/org/apache/hop/pipeline/transforms/formula/Formula.java
+++
b/plugins/transforms/formula/src/main/java/org/apache/hop/pipeline/transforms/formula/Formula.java
@@ -139,10 +139,8 @@ public class Formula extends BaseTransform<FormulaMeta,
FormulaData> {
}
Object[] outputRowData = RowDataUtil.resizeArray(r,
data.outputRowMeta.size());
- Object outputValue = null;
-
for (int i = 0; i < meta.getFormulas().size(); i++) {
-
+ Object outputValue = null;
FormulaMetaFunction formula = meta.getFormulas().get(i);
FormulaParser parser =
new FormulaParser(
diff --git
a/plugins/transforms/formula/src/test/java/org/apache/hop/pipeline/transforms/formula/FormulaTests.java
b/plugins/transforms/formula/src/test/java/org/apache/hop/pipeline/transforms/formula/FormulaTests.java
new file mode 100644
index 0000000000..0529284588
--- /dev/null
+++
b/plugins/transforms/formula/src/test/java/org/apache/hop/pipeline/transforms/formula/FormulaTests.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.formula;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.PipelineTestingUtil;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/** Unit test for {@link Formula} */
+class FormulaTests {
+ private TransformMockHelper<FormulaMeta, FormulaData> transformMockHelper;
+
+ @BeforeEach
+ void setUp() {
+ transformMockHelper =
+ new TransformMockHelper<>("Formula", FormulaMeta.class,
FormulaData.class);
+ when(transformMockHelper.logChannelFactory.create(any(),
any(ILoggingObject.class)))
+ .thenReturn(transformMockHelper.iLogChannel);
+ when(transformMockHelper.pipeline.isRunning()).thenReturn(true);
+ when(transformMockHelper.transformMeta.getName()).thenReturn("Formula");
+ }
+
+ @Test
+ void processRow() throws Exception {
+ Formula formula =
+ new Formula(
+ transformMockHelper.transformMeta,
+ transformMockHelper.iTransformMeta,
+ transformMockHelper.iTransformData,
+ 0,
+ transformMockHelper.pipelineMeta,
+ transformMockHelper.pipeline);
+
+ // init success.
+ boolean result = formula.init();
+ assertTrue(result);
+
+ // Set up input row meta
+ RowMeta inputRowMeta = new RowMeta();
+ inputRowMeta.addValueMeta(new ValueMetaInteger("Card"));
+ inputRowMeta.addValueMeta(new ValueMetaString("Privacy"));
+ formula.setInputRowMeta(inputRowMeta);
+
+ // spy
+ formula = spy(formula);
+ doReturn(new Object[] {29, null})
+ .doReturn(new Object[] {0, "Bob"})
+ .doReturn(null)
+ .when(formula)
+ .getRow();
+
+ List<Object[]> execCount = PipelineTestingUtil.execute(formula, 2, false);
+ assertEquals(2, execCount.size());
+
+ IRowMeta outputRowMeta = transformMockHelper.iTransformData.outputRowMeta;
+ assertEquals(2, outputRowMeta.size());
+
+ formula.dispose();
+ }
+
+ @AfterEach
+ void tearDown() {
+ transformMockHelper.cleanUp();
+ }
+}
