This is an automated email from the ASF dual-hosted git repository. hansva pushed a commit to branch 2.18.1-patch in repository https://gitbox.apache.org/repos/asf/hop.git
commit 827b7ed1bc0c0aa57481fb6af6fb658953a048be Author: Hans Van Akelyen <[email protected]> AuthorDate: Fri Jun 12 12:44:33 2026 +0200 keep original values of not injected, fixes #7275 (#7281) --- .../hop/metadata/inject/HopMetadataInjector.java | 14 +- .../metadata/inject/HopMetadataInjectorTest.java | 71 ++++++ .../mdi/0060-mdi-generate-rows-preserve-child.hpl | 104 ++++++++ .../mdi/0060-mdi-generate-rows-preserve-parent.hpl | 266 +++++++++++++++++++++ .../mdi/main-0060-mdi-generate-rows-preserve.hwf | 88 +++++++ 5 files changed, 539 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/hop/metadata/inject/HopMetadataInjector.java b/core/src/main/java/org/apache/hop/metadata/inject/HopMetadataInjector.java index c20ca7952a..1886834016 100644 --- a/core/src/main/java/org/apache/hop/metadata/inject/HopMetadataInjector.java +++ b/core/src/main/java/org/apache/hop/metadata/inject/HopMetadataInjector.java @@ -22,6 +22,7 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.ParameterizedType; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -522,6 +523,7 @@ public class HopMetadataInjector { // List<Object> list = (List<Object>) ReflectionUtil.findGetter(objectClass, field).invoke(object); + List<Object> existingItems = new ArrayList<>(list); list.clear(); // Which class is being listed? @@ -539,17 +541,20 @@ public class HopMetadataInjector { // The keys (and their types) are in the row metadata. // IRowMeta rowMeta = rowBuffer.getRowMeta(); + int rowIndex = 0; for (Object[] row : rowBuffer.getBuffer()) { // Is this a primitive type we're dealing with? // if (listItemClass.isPrimitive() || listItemClass.equals(String.class)) { // The row contains a single value that we simply need to add to the list - // list.add(row[0]); } else { - // Create a new list item for every row. - // - Object listItemObject = listItemClass.getConstructor().newInstance(); + // Reuse the pre-defined template item at the same position so its non-injected fields + // are preserved; only create a new item when injecting beyond the pre-defined rows. + Object listItemObject = + rowIndex < existingItems.size() + ? existingItems.get(rowIndex) + : listItemClass.getConstructor().newInstance(); // Add it to the list list.add(listItemObject); @@ -574,6 +579,7 @@ public class HopMetadataInjector { } } } + rowIndex++; } } catch (Exception e) { throw new HopException( diff --git a/core/src/test/java/org/apache/hop/metadata/inject/HopMetadataInjectorTest.java b/core/src/test/java/org/apache/hop/metadata/inject/HopMetadataInjectorTest.java index 704f3d2820..6cc480bede 100644 --- a/core/src/test/java/org/apache/hop/metadata/inject/HopMetadataInjectorTest.java +++ b/core/src/test/java/org/apache/hop/metadata/inject/HopMetadataInjectorTest.java @@ -21,6 +21,7 @@ package org.apache.hop.metadata.inject; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.HashMap; @@ -101,6 +102,76 @@ class HopMetadataInjectorTest { assertEquals("Mouse", e.getLastName()); } + /** + * Regression test for <a href="https://github.com/apache/hop/issues/7275";>#7275</a>: when a + * template list already holds pre-configured items and only a subset of the item's fields is + * injected, the fields that are not injected must keep their pre-defined values (merge by + * position) instead of being wiped. The legacy array-based injection left un-injected attributes + * untouched; the list rebuild used to clear everything. + */ + @Test + void injectListMergesWithPreConfiguredItems() throws Exception { + Company company = new Company(); + // Two pre-defined employees with BOTH first and last name set (as a template would). + Employee predefined1 = new Employee(); + predefined1.setFirstName("placeholder1"); + predefined1.setLastName("Duck"); + Employee predefined2 = new Employee(); + predefined2.setFirstName("placeholder2"); + predefined2.setLastName("Mouse"); + company.getEmployees().add(predefined1); + company.getEmployees().add(predefined2); + + // Inject ONLY the first name (last name is not part of the injected row buffer). + Map<String, Object> injectionKeyMap = new HashMap<>(); + Map<String, RowBuffer> injectionGroupMap = new HashMap<>(); + RowBuffer rowBuffer = new RowBuffer(); + rowBuffer.setRowMeta(new RowMetaBuilder().addString("FIRST_NAME").build()); + rowBuffer.addRow("Donald"); + rowBuffer.addRow("Micky"); + injectionGroupMap.put("EMPLOYEES", rowBuffer); + + HopMetadataInjector.inject( + new MemoryMetadataProvider(), company, injectionKeyMap, injectionGroupMap); + + assertEquals(2, company.getEmployees().size()); + // First name is overwritten by injection... + assertEquals("Donald", company.getEmployees().get(0).getFirstName()); + assertEquals("Micky", company.getEmployees().get(1).getFirstName()); + // ...but the un-injected last name keeps the pre-defined value. + assertEquals("Duck", company.getEmployees().get(0).getLastName()); + assertEquals("Mouse", company.getEmployees().get(1).getLastName()); + } + + /** + * When the injection provides more rows than the template pre-defines, the extra rows are added + * as new items (so adding lines beyond the pre-defined ones still works). + */ + @Test + void injectListExtendsBeyondPreConfiguredItems() throws Exception { + Company company = new Company(); + Employee predefined = new Employee(); + predefined.setFirstName("placeholder"); + predefined.setLastName("Keep"); + company.getEmployees().add(predefined); + + Map<String, RowBuffer> injectionGroupMap = new HashMap<>(); + RowBuffer rowBuffer = new RowBuffer(); + rowBuffer.setRowMeta(new RowMetaBuilder().addString("FIRST_NAME").build()); + rowBuffer.addRow("Donald"); + rowBuffer.addRow("Micky"); // beyond the single pre-defined item + injectionGroupMap.put("EMPLOYEES", rowBuffer); + + HopMetadataInjector.inject( + new MemoryMetadataProvider(), company, new HashMap<>(), injectionGroupMap); + + assertEquals(2, company.getEmployees().size()); + assertEquals("Donald", company.getEmployees().get(0).getFirstName()); + assertEquals("Keep", company.getEmployees().get(0).getLastName()); // preserved + assertEquals("Micky", company.getEmployees().get(1).getFirstName()); + assertNull(company.getEmployees().get(1).getLastName()); // brand new item + } + @Test void testCompanyMapping() throws Exception { Map<String, Set<String>> map = HopMetadataInjector.findInjectionGroupKeys(Company.class); diff --git a/integration-tests/mdi/0060-mdi-generate-rows-preserve-child.hpl b/integration-tests/mdi/0060-mdi-generate-rows-preserve-child.hpl new file mode 100644 index 0000000000..3375b59769 --- /dev/null +++ b/integration-tests/mdi/0060-mdi-generate-rows-preserve-child.hpl @@ -0,0 +1,104 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +--> +<pipeline> + <info> + <name>0060-mdi-generate-rows-preserve-child</name> + <name_sync_with_filename>Y</name_sync_with_filename> + <description>Template (issue #7275): Generate Rows pre-defines Name and Type for two fields; only the Value is injected by the parent.</description> + <pipeline_type>Normal</pipeline_type> + <parameters> + </parameters> + </info> + <order> + <hop> + <from>Generate rows</from> + <to>out</to> + <enabled>Y</enabled> + </hop> + </order> + <transform> + <name>Generate rows</name> + <type>RowGenerator</type> + <description/> + <distribute>Y</distribute> + <custom_distribution/> + <copies>1</copies> + <partitioning> + <method>none</method> + <schema_name/> + </partitioning> + <fields> + <field> + <currency/> + <decimal/> + <format/> + <group/> + <length>-1</length> + <name>val_a</name> + <precision>-1</precision> + <set_empty_string>N</set_empty_string> + <type>String</type> + <nullif>PLACEHOLDER_A</nullif> + </field> + <field> + <currency/> + <decimal/> + <format/> + <group/> + <length>-1</length> + <name>val_b</name> + <precision>-1</precision> + <set_empty_string>N</set_empty_string> + <type>String</type> + <nullif>PLACEHOLDER_B</nullif> + </field> + </fields> + <interval_in_ms>5000</interval_in_ms> + <last_time_field>FiveSecondsAgo</last_time_field> + <never_ending>N</never_ending> + <limit>1</limit> + <row_time_field>now</row_time_field> + <attributes/> + <GUI> + <xloc>176</xloc> + <yloc>96</yloc> + </GUI> + </transform> + <transform> + <name>out</name> + <type>Dummy</type> + <description/> + <distribute>Y</distribute> + <custom_distribution/> + <copies>1</copies> + <partitioning> + <method>none</method> + <schema_name/> + </partitioning> + <attributes/> + <GUI> + <xloc>368</xloc> + <yloc>96</yloc> + </GUI> + </transform> + <transform_error_handling> + </transform_error_handling> + <attributes/> +</pipeline> diff --git a/integration-tests/mdi/0060-mdi-generate-rows-preserve-parent.hpl b/integration-tests/mdi/0060-mdi-generate-rows-preserve-parent.hpl new file mode 100644 index 0000000000..11fc8192b2 --- /dev/null +++ b/integration-tests/mdi/0060-mdi-generate-rows-preserve-parent.hpl @@ -0,0 +1,266 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +--> +<pipeline> + <info> + <capture_transform_performance>N</capture_transform_performance> + <transform_performance_capturing_delay>1000</transform_performance_capturing_delay> + <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit> + <pipeline_type>Normal</pipeline_type> + <pipeline_status>0</pipeline_status> + <parameters/> + <name>0060-mdi-generate-rows-preserve-parent</name> + <name_sync_with_filename>Y</name_sync_with_filename> + <description>Issue #7275: inject only the Value into a Generate Rows template and verify the pre-defined Name/Type fields are preserved (merged), not wiped.</description> + <created_user>-</created_user> + <modified_user>-</modified_user> + <created_date>2026/06/12 11:41:41.761</created_date> + <modified_date>2026/06/12 11:41:41.761</modified_date> + </info> + <transform> + <type>DataGrid</type> + <name>Value source</name> + <fields> + <field> + <currency/> + <decimal/> + <group/> + <name>v</name> + <type>String</type> + <format/> + <length>-1</length> + <precision>-1</precision> + <set_empty_string>N</set_empty_string> + </field> + </fields> + <data> + <line> + <item>AAA</item> + </line> + <line> + <item>BBB</item> + </line> + </data> + <distribute>Y</distribute> + <copies>1</copies> + <GUI> + <xloc>144</xloc> + <yloc>96</yloc> + </GUI> + <description/> + <partitioning> + <method>none</method> + <schema_name/> + </partitioning> + <attributes/> + </transform> + <transform> + <type>MetaInject</type> + <name>ETL metadata injection</name> + <filename>${PROJECT_HOME}/0060-mdi-generate-rows-preserve-child.hpl</filename> + <source_transform>out</source_transform> + <source_output_fields> + <source_output_field> + <source_output_field_name>val_a</source_output_field_name> + <source_output_field_type>String</source_output_field_type> + <source_output_field_length>-1</source_output_field_length> + <source_output_field_precision>-1</source_output_field_precision> + </source_output_field> + <source_output_field> + <source_output_field_name>val_b</source_output_field_name> + <source_output_field_type>String</source_output_field_type> + <source_output_field_length>-1</source_output_field_length> + <source_output_field_precision>-1</source_output_field_precision> + </source_output_field> + </source_output_fields> + <mappings> + <mapping> + <source_transform>Value source</source_transform> + <source_field>v</source_field> + <target_transform_name>Generate rows</target_transform_name> + <target_attribute_key>nullif</target_attribute_key> + <target_detail>Y</target_detail> + </mapping> + </mappings> + <target_file/> + <no_execution>N</no_execution> + <allow_empty_stream_on_execution>N</allow_empty_stream_on_execution> + <stream_source_transform/> + <stream_target_transform/> + <run_configuration>local</run_configuration> + <create_parent_folder>Y</create_parent_folder> + <distribute>Y</distribute> + <copies>1</copies> + <GUI> + <xloc>336</xloc> + <yloc>96</yloc> + </GUI> + <description/> + <partitioning> + <method>none</method> + <schema_name/> + </partitioning> + <attributes/> + </transform> + <transform> + <type>FilterRows</type> + <name>Check val_a</name> + <compare> + <condition> + <negated>N</negated> + <operator>-</operator> + <leftvalue>val_a</leftvalue> + <function>=</function> + <rightvalue/> + <value> + <name>constant</name> + <type>String</type> + <text>AAA</text> + <length>-1</length> + <precision>-1</precision> + <isnull>N</isnull> + <mask/> + </value> + <conditions/> + </condition> + </compare> + <send_true_to>Check val_b</send_true_to> + <send_false_to>Name or Type was wiped - Abort</send_false_to> + <distribute>Y</distribute> + <copies>1</copies> + <GUI> + <xloc>528</xloc> + <yloc>96</yloc> + </GUI> + <description/> + <partitioning> + <method>none</method> + <schema_name/> + </partitioning> + <attributes/> + </transform> + <transform> + <type>FilterRows</type> + <name>Check val_b</name> + <compare> + <condition> + <negated>N</negated> + <operator>-</operator> + <leftvalue>val_b</leftvalue> + <function>=</function> + <rightvalue/> + <value> + <name>constant</name> + <type>String</type> + <text>BBB</text> + <length>-1</length> + <precision>-1</precision> + <isnull>N</isnull> + <mask/> + </value> + <conditions/> + </condition> + </compare> + <send_true_to>Preserved OK</send_true_to> + <send_false_to>Name or Type was wiped - Abort</send_false_to> + <distribute>Y</distribute> + <copies>1</copies> + <GUI> + <xloc>720</xloc> + <yloc>96</yloc> + </GUI> + <description/> + <partitioning> + <method>none</method> + <schema_name/> + </partitioning> + <attributes/> + </transform> + <transform> + <type>Abort</type> + <name>Name or Type was wiped - Abort</name> + <row_threshold>0</row_threshold> + <message>Injecting only the Value wiped the pre-defined Name/Type fields of the Generate Rows template (issue #7275)</message> + <always_log_rows>Y</always_log_rows> + <abort_option>ABORT_WITH_ERROR</abort_option> + <distribute>Y</distribute> + <copies>1</copies> + <GUI> + <xloc>528</xloc> + <yloc>256</yloc> + </GUI> + <description/> + <partitioning> + <method>none</method> + <schema_name/> + </partitioning> + <attributes/> + </transform> + <transform> + <type>Dummy</type> + <name>Preserved OK</name> + <distribute>Y</distribute> + <copies>1</copies> + <GUI> + <xloc>912</xloc> + <yloc>96</yloc> + </GUI> + <description/> + <partitioning> + <method>none</method> + <schema_name/> + </partitioning> + <attributes/> + </transform> + <order> + <hop> + <from>Value source</from> + <to>ETL metadata injection</to> + <enabled>Y</enabled> + </hop> + <hop> + <from>ETL metadata injection</from> + <to>Check val_a</to> + <enabled>Y</enabled> + </hop> + <hop> + <from>Check val_a</from> + <to>Check val_b</to> + <enabled>Y</enabled> + </hop> + <hop> + <from>Check val_a</from> + <to>Name or Type was wiped - Abort</to> + <enabled>Y</enabled> + </hop> + <hop> + <from>Check val_b</from> + <to>Preserved OK</to> + <enabled>Y</enabled> + </hop> + <hop> + <from>Check val_b</from> + <to>Name or Type was wiped - Abort</to> + <enabled>Y</enabled> + </hop> + </order> + <notepads/> + <attributes/> + <transform_error_handling/> +</pipeline> diff --git a/integration-tests/mdi/main-0060-mdi-generate-rows-preserve.hwf b/integration-tests/mdi/main-0060-mdi-generate-rows-preserve.hwf new file mode 100644 index 0000000000..fb28c25aea --- /dev/null +++ b/integration-tests/mdi/main-0060-mdi-generate-rows-preserve.hwf @@ -0,0 +1,88 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +--> +<workflow> + <name>main-0060-mdi-generate-rows-preserve</name> + <name_sync_with_filename>Y</name_sync_with_filename> + <description/> + <created_user>-</created_user> + <created_date>2026/06/12 12:00:00.000</created_date> + <modified_user>-</modified_user> + <modified_date>2026/06/12 12:00:00.000</modified_date> + <parameters> + </parameters> + <actions> + <action> + <name>Start</name> + <description/> + <type>SPECIAL</type> + <attributes/> + <repeat>N</repeat> + <schedulerType>0</schedulerType> + <intervalSeconds>0</intervalSeconds> + <intervalMinutes>60</intervalMinutes> + <hour>12</hour> + <minutes>0</minutes> + <weekDay>1</weekDay> + <DayOfMonth>1</DayOfMonth> + <parallel>N</parallel> + <xloc>128</xloc> + <yloc>112</yloc> + <attributes_hac/> + </action> + <action> + <name>0060-mdi-generate-rows-preserve-parent.hpl</name> + <description/> + <type>PIPELINE</type> + <attributes/> + <add_date>N</add_date> + <add_time>N</add_time> + <clear_files>N</clear_files> + <clear_rows>N</clear_rows> + <create_parent_folder>N</create_parent_folder> + <exec_per_row>N</exec_per_row> + <filename>${PROJECT_HOME}/0060-mdi-generate-rows-preserve-parent.hpl</filename> + <loglevel>Basic</loglevel> + <parameters> + <pass_all_parameters>Y</pass_all_parameters> + </parameters> + <params_from_previous>N</params_from_previous> + <run_configuration>local</run_configuration> + <set_append_logfile>N</set_append_logfile> + <set_logfile>N</set_logfile> + <wait_until_finished>Y</wait_until_finished> + <parallel>N</parallel> + <xloc>320</xloc> + <yloc>112</yloc> + <attributes_hac/> + </action> + </actions> + <hops> + <hop> + <from>Start</from> + <to>0060-mdi-generate-rows-preserve-parent.hpl</to> + <enabled>Y</enabled> + <evaluation>Y</evaluation> + <unconditional>Y</unconditional> + </hop> + </hops> + <notepads> + </notepads> + <attributes/> +</workflow>
