This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new a49b8cecbf keep original values of not injected, fixes #7275 (#7281)
a49b8cecbf is described below
commit a49b8cecbfc79080d0aa6c7b26b0189161fe31be
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Fri Jun 12 12:44:33 2026 +0200
keep original values of not injected, fixes #7275 (#7281)
---
.../hop/metadata/inject/HopMetadataInjector.java | 14 +-
.../metadata/inject/HopMetadataInjectorTest.java | 71 ++++++
.../mdi/0060-mdi-generate-rows-preserve-child.hpl | 104 ++++++++
.../mdi/0060-mdi-generate-rows-preserve-parent.hpl | 266 +++++++++++++++++++++
.../mdi/main-0060-mdi-generate-rows-preserve.hwf | 88 +++++++
5 files changed, 539 insertions(+), 4 deletions(-)
diff --git
a/core/src/main/java/org/apache/hop/metadata/inject/HopMetadataInjector.java
b/core/src/main/java/org/apache/hop/metadata/inject/HopMetadataInjector.java
index c20ca7952a..1886834016 100644
--- a/core/src/main/java/org/apache/hop/metadata/inject/HopMetadataInjector.java
+++ b/core/src/main/java/org/apache/hop/metadata/inject/HopMetadataInjector.java
@@ -22,6 +22,7 @@ import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.ParameterizedType;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -522,6 +523,7 @@ public class HopMetadataInjector {
//
List<Object> list =
(List<Object>) ReflectionUtil.findGetter(objectClass,
field).invoke(object);
+ List<Object> existingItems = new ArrayList<>(list);
list.clear();
// Which class is being listed?
@@ -539,17 +541,20 @@ public class HopMetadataInjector {
// The keys (and their types) are in the row metadata.
//
IRowMeta rowMeta = rowBuffer.getRowMeta();
+ int rowIndex = 0;
for (Object[] row : rowBuffer.getBuffer()) {
// Is this a primitive type we're dealing with?
//
if (listItemClass.isPrimitive() || listItemClass.equals(String.class))
{
// The row contains a single value that we simply need to add to the
list
- //
list.add(row[0]);
} else {
- // Create a new list item for every row.
- //
- Object listItemObject = listItemClass.getConstructor().newInstance();
+ // Reuse the pre-defined template item at the same position so its
non-injected fields
+ // are preserved; only create a new item when injecting beyond the
pre-defined rows.
+ Object listItemObject =
+ rowIndex < existingItems.size()
+ ? existingItems.get(rowIndex)
+ : listItemClass.getConstructor().newInstance();
// Add it to the list
list.add(listItemObject);
@@ -574,6 +579,7 @@ public class HopMetadataInjector {
}
}
}
+ rowIndex++;
}
} catch (Exception e) {
throw new HopException(
diff --git
a/core/src/test/java/org/apache/hop/metadata/inject/HopMetadataInjectorTest.java
b/core/src/test/java/org/apache/hop/metadata/inject/HopMetadataInjectorTest.java
index 704f3d2820..6cc480bede 100644
---
a/core/src/test/java/org/apache/hop/metadata/inject/HopMetadataInjectorTest.java
+++
b/core/src/test/java/org/apache/hop/metadata/inject/HopMetadataInjectorTest.java
@@ -21,6 +21,7 @@ package org.apache.hop.metadata.inject;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.HashMap;
@@ -101,6 +102,76 @@ class HopMetadataInjectorTest {
assertEquals("Mouse", e.getLastName());
}
+ /**
+ * Regression test for <a
href="https://github.com/apache/hop/issues/7275";>#7275</a>: when a
+ * template list already holds pre-configured items and only a subset of the
item's fields is
+ * injected, the fields that are not injected must keep their pre-defined
values (merge by
+ * position) instead of being wiped. The legacy array-based injection left
un-injected attributes
+ * untouched; the list rebuild used to clear everything.
+ */
+ @Test
+ void injectListMergesWithPreConfiguredItems() throws Exception {
+ Company company = new Company();
+ // Two pre-defined employees with BOTH first and last name set (as a
template would).
+ Employee predefined1 = new Employee();
+ predefined1.setFirstName("placeholder1");
+ predefined1.setLastName("Duck");
+ Employee predefined2 = new Employee();
+ predefined2.setFirstName("placeholder2");
+ predefined2.setLastName("Mouse");
+ company.getEmployees().add(predefined1);
+ company.getEmployees().add(predefined2);
+
+ // Inject ONLY the first name (last name is not part of the injected row
buffer).
+ Map<String, Object> injectionKeyMap = new HashMap<>();
+ Map<String, RowBuffer> injectionGroupMap = new HashMap<>();
+ RowBuffer rowBuffer = new RowBuffer();
+ rowBuffer.setRowMeta(new RowMetaBuilder().addString("FIRST_NAME").build());
+ rowBuffer.addRow("Donald");
+ rowBuffer.addRow("Micky");
+ injectionGroupMap.put("EMPLOYEES", rowBuffer);
+
+ HopMetadataInjector.inject(
+ new MemoryMetadataProvider(), company, injectionKeyMap,
injectionGroupMap);
+
+ assertEquals(2, company.getEmployees().size());
+ // First name is overwritten by injection...
+ assertEquals("Donald", company.getEmployees().get(0).getFirstName());
+ assertEquals("Micky", company.getEmployees().get(1).getFirstName());
+ // ...but the un-injected last name keeps the pre-defined value.
+ assertEquals("Duck", company.getEmployees().get(0).getLastName());
+ assertEquals("Mouse", company.getEmployees().get(1).getLastName());
+ }
+
+ /**
+ * When the injection provides more rows than the template pre-defines, the
extra rows are added
+ * as new items (so adding lines beyond the pre-defined ones still works).
+ */
+ @Test
+ void injectListExtendsBeyondPreConfiguredItems() throws Exception {
+ Company company = new Company();
+ Employee predefined = new Employee();
+ predefined.setFirstName("placeholder");
+ predefined.setLastName("Keep");
+ company.getEmployees().add(predefined);
+
+ Map<String, RowBuffer> injectionGroupMap = new HashMap<>();
+ RowBuffer rowBuffer = new RowBuffer();
+ rowBuffer.setRowMeta(new RowMetaBuilder().addString("FIRST_NAME").build());
+ rowBuffer.addRow("Donald");
+ rowBuffer.addRow("Micky"); // beyond the single pre-defined item
+ injectionGroupMap.put("EMPLOYEES", rowBuffer);
+
+ HopMetadataInjector.inject(
+ new MemoryMetadataProvider(), company, new HashMap<>(),
injectionGroupMap);
+
+ assertEquals(2, company.getEmployees().size());
+ assertEquals("Donald", company.getEmployees().get(0).getFirstName());
+ assertEquals("Keep", company.getEmployees().get(0).getLastName()); //
preserved
+ assertEquals("Micky", company.getEmployees().get(1).getFirstName());
+ assertNull(company.getEmployees().get(1).getLastName()); // brand new item
+ }
+
@Test
void testCompanyMapping() throws Exception {
Map<String, Set<String>> map =
HopMetadataInjector.findInjectionGroupKeys(Company.class);
diff --git a/integration-tests/mdi/0060-mdi-generate-rows-preserve-child.hpl
b/integration-tests/mdi/0060-mdi-generate-rows-preserve-child.hpl
new file mode 100644
index 0000000000..3375b59769
--- /dev/null
+++ b/integration-tests/mdi/0060-mdi-generate-rows-preserve-child.hpl
@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0060-mdi-generate-rows-preserve-child</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Template (issue #7275): Generate Rows pre-defines Name and
Type for two fields; only the Value is injected by the parent.</description>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ </info>
+ <order>
+ <hop>
+ <from>Generate rows</from>
+ <to>out</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Generate rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>val_a</name>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <type>String</type>
+ <nullif>PLACEHOLDER_A</nullif>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>val_b</name>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <type>String</type>
+ <nullif>PLACEHOLDER_B</nullif>
+ </field>
+ </fields>
+ <interval_in_ms>5000</interval_in_ms>
+ <last_time_field>FiveSecondsAgo</last_time_field>
+ <never_ending>N</never_ending>
+ <limit>1</limit>
+ <row_time_field>now</row_time_field>
+ <attributes/>
+ <GUI>
+ <xloc>176</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>out</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>368</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/mdi/0060-mdi-generate-rows-preserve-parent.hpl
b/integration-tests/mdi/0060-mdi-generate-rows-preserve-parent.hpl
new file mode 100644
index 0000000000..11fc8192b2
--- /dev/null
+++ b/integration-tests/mdi/0060-mdi-generate-rows-preserve-parent.hpl
@@ -0,0 +1,266 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters/>
+ <name>0060-mdi-generate-rows-preserve-parent</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Issue #7275: inject only the Value into a Generate Rows
template and verify the pre-defined Name/Type fields are preserved (merged),
not wiped.</description>
+ <created_user>-</created_user>
+ <modified_user>-</modified_user>
+ <created_date>2026/06/12 11:41:41.761</created_date>
+ <modified_date>2026/06/12 11:41:41.761</modified_date>
+ </info>
+ <transform>
+ <type>DataGrid</type>
+ <name>Value source</name>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <group/>
+ <name>v</name>
+ <type>String</type>
+ <format/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>AAA</item>
+ </line>
+ <line>
+ <item>BBB</item>
+ </line>
+ </data>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>144</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>MetaInject</type>
+ <name>ETL metadata injection</name>
+
<filename>${PROJECT_HOME}/0060-mdi-generate-rows-preserve-child.hpl</filename>
+ <source_transform>out</source_transform>
+ <source_output_fields>
+ <source_output_field>
+ <source_output_field_name>val_a</source_output_field_name>
+ <source_output_field_type>String</source_output_field_type>
+ <source_output_field_length>-1</source_output_field_length>
+ <source_output_field_precision>-1</source_output_field_precision>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_name>val_b</source_output_field_name>
+ <source_output_field_type>String</source_output_field_type>
+ <source_output_field_length>-1</source_output_field_length>
+ <source_output_field_precision>-1</source_output_field_precision>
+ </source_output_field>
+ </source_output_fields>
+ <mappings>
+ <mapping>
+ <source_transform>Value source</source_transform>
+ <source_field>v</source_field>
+ <target_transform_name>Generate rows</target_transform_name>
+ <target_attribute_key>nullif</target_attribute_key>
+ <target_detail>Y</target_detail>
+ </mapping>
+ </mappings>
+ <target_file/>
+ <no_execution>N</no_execution>
+ <allow_empty_stream_on_execution>N</allow_empty_stream_on_execution>
+ <stream_source_transform/>
+ <stream_target_transform/>
+ <run_configuration>local</run_configuration>
+ <create_parent_folder>Y</create_parent_folder>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>336</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>FilterRows</type>
+ <name>Check val_a</name>
+ <compare>
+ <condition>
+ <negated>N</negated>
+ <operator>-</operator>
+ <leftvalue>val_a</leftvalue>
+ <function>=</function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>String</type>
+ <text>AAA</text>
+ <length>-1</length>
+ <precision>-1</precision>
+ <isnull>N</isnull>
+ <mask/>
+ </value>
+ <conditions/>
+ </condition>
+ </compare>
+ <send_true_to>Check val_b</send_true_to>
+ <send_false_to>Name or Type was wiped - Abort</send_false_to>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>528</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>FilterRows</type>
+ <name>Check val_b</name>
+ <compare>
+ <condition>
+ <negated>N</negated>
+ <operator>-</operator>
+ <leftvalue>val_b</leftvalue>
+ <function>=</function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>String</type>
+ <text>BBB</text>
+ <length>-1</length>
+ <precision>-1</precision>
+ <isnull>N</isnull>
+ <mask/>
+ </value>
+ <conditions/>
+ </condition>
+ </compare>
+ <send_true_to>Preserved OK</send_true_to>
+ <send_false_to>Name or Type was wiped - Abort</send_false_to>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>720</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>Abort</type>
+ <name>Name or Type was wiped - Abort</name>
+ <row_threshold>0</row_threshold>
+ <message>Injecting only the Value wiped the pre-defined Name/Type fields
of the Generate Rows template (issue #7275)</message>
+ <always_log_rows>Y</always_log_rows>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>528</xloc>
+ <yloc>256</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>Dummy</type>
+ <name>Preserved OK</name>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>912</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <order>
+ <hop>
+ <from>Value source</from>
+ <to>ETL metadata injection</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>ETL metadata injection</from>
+ <to>Check val_a</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Check val_a</from>
+ <to>Check val_b</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Check val_a</from>
+ <to>Name or Type was wiped - Abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Check val_b</from>
+ <to>Preserved OK</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Check val_b</from>
+ <to>Name or Type was wiped - Abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <notepads/>
+ <attributes/>
+ <transform_error_handling/>
+</pipeline>
diff --git a/integration-tests/mdi/main-0060-mdi-generate-rows-preserve.hwf
b/integration-tests/mdi/main-0060-mdi-generate-rows-preserve.hwf
new file mode 100644
index 0000000000..fb28c25aea
--- /dev/null
+++ b/integration-tests/mdi/main-0060-mdi-generate-rows-preserve.hwf
@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0060-mdi-generate-rows-preserve</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <created_user>-</created_user>
+ <created_date>2026/06/12 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/06/12 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <xloc>128</xloc>
+ <yloc>112</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0060-mdi-generate-rows-preserve-parent.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+
<filename>${PROJECT_HOME}/0060-mdi-generate-rows-preserve-parent.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>320</xloc>
+ <yloc>112</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0060-mdi-generate-rows-preserve-parent.hpl</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
