This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 1033a5fd54 issue #6369 (#6726)
1033a5fd54 is described below
commit 1033a5fd549887152b0e90f1d53b8ceb09c0ba6c
Author: Matt Casters <[email protected]>
AuthorDate: Sat Mar 7 09:17:50 2026 +0100
issue #6369 (#6726)
* issue #6369
* issue #6369 (cosmetic)
---
.../ROOT/pages/pipeline/transforms/mergerows.adoc | 15 +-
.../transforms/0077-merge-rows-passthrough.hpl | 322 +++++++++++++++++
.../datasets/golden-merge-rows-passthrough.csv | 13 +
.../transforms/main-0077-merge-rows.hwf | 3 +
.../dataset/golden-merge-rows-passthrough.json | 88 +++++
.../0077-merge-rows-passthrough UNIT.json | 64 ++++
.../pipeline/transforms/mergerows/MergeRows.java | 122 +++++--
.../transforms/mergerows/MergeRowsData.java | 7 +
.../transforms/mergerows/MergeRowsDialog.java | 399 +++++++++++++++------
.../transforms/mergerows/MergeRowsMeta.java | 47 +++
.../{MergeRowsData.java => PassThroughField.java} | 38 +-
.../mergerows/messages/messages_en_US.properties | 13 +-
12 files changed, 976 insertions(+), 155 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/mergerows.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/mergerows.adoc
index f8d8b317e6..a6e451b24f 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/mergerows.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/mergerows.adoc
@@ -75,14 +75,21 @@ In the subsequent transform, you can use the flag field
generated by **Merge row
|Transform name|Name of the transform.
|Reference rows origin|Specify the transform that produces the reference rows.
It's a Stream with original rows (rows that you want to compare the new rows
to).
|Compare rows origin|Specify the transform that produces the compare rows.
It's a Stream with new rows
-|Flag fieldname|Specify the name of the flag field on the output stream.
-|Difference fieldname|Specify the name of the field to contain the difference
in case the status is **changed**.
+|Flag field name|Specify the name of the flag field on the output stream.
+|Difference field name|Specify the name of the field to contain the difference
in case the status is **changed**.
The difference will be in the form of a "changes" array in JSON format.
-|Keys to match|Specify fields containing the keys on which to match. Click
"Get key fields" to insert all of the fields from the reference rows
-|Values to compare|Specify fields contaning the values to compare. Click "Get
value fields" to insert all of the fields from the compare rows.
+|Keys to match|Specify fields containing the keys on which to match. Click
"Get key fields" to insert all the fields from the reference rows
+|Values to compare|Specify fields containing the values to compare. Click "Get
value fields" to insert all the fields from the compare rows.
Key fields do not need to be repeated here.
|===
+== Passing through fields
+
+It's possible to pass through fields from the reference or compare data
streams.
+To do this select the "Extra" tab and specify the fields you need to retain
from either source transform. You can use the `[Get fields to pass through]`
button to add all possible source fields. You can then select the rows to keep
in the table view and use the "Keep" toolbar button (CTRL-K).
+
+NOTE: for `new` or `deleted` rows obviously only either reference or compare
values are available. The other ones will always be `null`.
+
== Differences JSON example
If you specify a field name for the differences in JSON, you will changes
appear when a change was detected:
diff --git a/integration-tests/transforms/0077-merge-rows-passthrough.hpl
b/integration-tests/transforms/0077-merge-rows-passthrough.hpl
new file mode 100644
index 0000000000..5b563396a5
--- /dev/null
+++ b/integration-tests/transforms/0077-merge-rows-passthrough.hpl
@@ -0,0 +1,322 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0077-merge-rows-passthrough</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2025/09/02 10:00:26.808</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2025/09/02 10:00:26.808</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>source1</from>
+ <to>Merge rows (diff)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>source2</from>
+ <to>Merge rows (diff)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Merge rows (diff)</from>
+ <to>validate</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Merge rows (diff)</name>
+ <type>MergeRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <compare>source2</compare>
+ <flag_field>flagfield</flag_field>
+ <keys>
+ <key>id</key>
+ </keys>
+ <passthrough-fields>
+ <passthrough-field>
+ <referenceField>Y</referenceField>
+ <renameTo>ref-id</renameTo>
+ <sourceField>id</sourceField>
+ </passthrough-field>
+ <passthrough-field>
+ <referenceField>Y</referenceField>
+ <renameTo>ref-name</renameTo>
+ <sourceField>name</sourceField>
+ </passthrough-field>
+ <passthrough-field>
+ <referenceField>Y</referenceField>
+ <renameTo>ref-score</renameTo>
+ <sourceField>score</sourceField>
+ </passthrough-field>
+ <passthrough-field>
+ <referenceField>N</referenceField>
+ <renameTo>cmp-id</renameTo>
+ <sourceField>id</sourceField>
+ </passthrough-field>
+ <passthrough-field>
+ <referenceField>N</referenceField>
+ <renameTo>cmp-name</renameTo>
+ <sourceField>name</sourceField>
+ </passthrough-field>
+ <passthrough-field>
+ <referenceField>N</referenceField>
+ <renameTo>cmp-score</renameTo>
+ <sourceField>score</sourceField>
+ </passthrough-field>
+ </passthrough-fields>
+ <reference>source1</reference>
+ <values>
+ <value>name</value>
+ <value>score</value>
+ </values>
+ <attributes/>
+ <GUI>
+ <xloc>240</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>source1</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>1</item>
+ <item>James</item>
+ <item>3.6</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>Marie</item>
+ <item>8.3</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item>Michael</item>
+ <item>9.1</item>
+ </line>
+ <line>
+ <item>4</item>
+ <item>Patrcia</item>
+ <item>4.2</item>
+ </line>
+ <line>
+ <item>5</item>
+ <item>Robert</item>
+ <item>5.0</item>
+ </line>
+ <line>
+ <item>6</item>
+ <item>Linda</item>
+ <item>1.6</item>
+ </line>
+ <line>
+ <item>7</item>
+ <item>David</item>
+ <item>7.2</item>
+ </line>
+ <line>
+ <item>8</item>
+ <item>Elizabeth</item>
+ <item>6.9</item>
+ </line>
+ <line>
+ <item>9</item>
+ <item>Willian</item>
+ <item>4.2</item>
+ </line>
+ <line>
+ <item>10</item>
+ <item>Barbara</item>
+ <item>0.5</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>id</name>
+ <type>Integer</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>name</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>score</name>
+ <format>#.#</format>
+ <type>Number</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>48</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>source2</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>1</item>
+ <item>James</item>
+ <item>3.6</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item>Michael</item>
+ <item>9.1</item>
+ </line>
+ <line>
+ <item>4</item>
+ <item>Patricia</item>
+ <item>4.2</item>
+ </line>
+ <line>
+ <item>5</item>
+ <item>Robert</item>
+ <item>5.0</item>
+ </line>
+ <line>
+ <item>7</item>
+ <item>David</item>
+ <item>7.2</item>
+ </line>
+ <line>
+ <item>8</item>
+ <item>Elizabeth</item>
+ <item>6.9</item>
+ </line>
+ <line>
+ <item>9</item>
+ <item>William</item>
+ <item>4.2</item>
+ </line>
+ <line>
+ <item>10</item>
+ <item>Barbara</item>
+ <item>0.5</item>
+ </line>
+ <line>
+ <item>11</item>
+ <item>Richard</item>
+ <item>9.6</item>
+ </line>
+ <line>
+ <item>12</item>
+ <item>Susan</item>
+ <item>5.4</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>id</name>
+ <type>Integer</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>name</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>score</name>
+ <format>#.#</format>
+ <type>Number</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>384</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-merge-rows-passthrough.csv
b/integration-tests/transforms/datasets/golden-merge-rows-passthrough.csv
new file mode 100644
index 0000000000..e8a9fec66e
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-merge-rows-passthrough.csv
@@ -0,0 +1,13 @@
+id,name,score,flagfield,ref-id,ref-name,ref-score,cmp-id,cmp-name,cmp-score
+1,James,3.6,identical,1,James,3.6,1,James,3.6
+2,Marie,8.3,deleted,2,Marie,8.3,,,
+3,Michael,9.1,identical,3,Michael,9.1,3,Michael,9.1
+4,Patricia,4.2,changed,4,Patrcia,4.2,4,Patricia,4.2
+5,Robert,5,identical,5,Robert,5,5,Robert,5
+6,Linda,1.6,deleted,6,Linda,1.6,,,
+7,David,7.2,identical,7,David,7.2,7,David,7.2
+8,Elizabeth,6.9,identical,8,Elizabeth,6.9,8,Elizabeth,6.9
+9,William,4.2,changed,9,Willian,4.2,9,William,4.2
+10,Barbara,0.5,identical,10,Barbara,0.5,10,Barbara,0.5
+11,Richard,9.6,new,,,,11,Richard,9.6
+12,Susan,5.4,new,,,,12,Susan,5.4
diff --git a/integration-tests/transforms/main-0077-merge-rows.hwf
b/integration-tests/transforms/main-0077-merge-rows.hwf
index 3a2878fbad..381b71ece9 100644
--- a/integration-tests/transforms/main-0077-merge-rows.hwf
+++ b/integration-tests/transforms/main-0077-merge-rows.hwf
@@ -61,6 +61,9 @@ limitations under the License.
<test_name>
<name>0077-merge-rows-diff UNIT</name>
</test_name>
+ <test_name>
+ <name>0077-merge-rows-passthrough UNIT</name>
+ </test_name>
</test_names>
<parallel>N</parallel>
<xloc>272</xloc>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-merge-rows-passthrough.json
b/integration-tests/transforms/metadata/dataset/golden-merge-rows-passthrough.json
new file mode 100644
index 0000000000..4c88ab3e7d
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-merge-rows-passthrough.json
@@ -0,0 +1,88 @@
+{
+ "base_filename": "golden-merge-rows-passthrough.csv",
+ "name": "golden-merge-rows-passthrough",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "id",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 1,
+ "field_precision": -1,
+ "field_name": "score",
+ "field_format": "#.#"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "flagfield",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "ref-id",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "ref-name",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 1,
+ "field_precision": -1,
+ "field_name": "ref-score",
+ "field_format": "#.#"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "cmp-id",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "cmp-name",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 1,
+ "field_precision": -1,
+ "field_name": "cmp-score",
+ "field_format": "#.#"
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/unit-test/0077-merge-rows-passthrough
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0077-merge-rows-passthrough
UNIT.json
new file mode 100644
index 0000000000..bf6e8c88cd
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0077-merge-rows-passthrough
UNIT.json
@@ -0,0 +1,64 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "name",
+ "data_set_field": "name"
+ },
+ {
+ "transform_field": "score",
+ "data_set_field": "score"
+ },
+ {
+ "transform_field": "flagfield",
+ "data_set_field": "flagfield"
+ },
+ {
+ "transform_field": "ref-id",
+ "data_set_field": "ref-id"
+ },
+ {
+ "transform_field": "ref-name",
+ "data_set_field": "ref-name"
+ },
+ {
+ "transform_field": "ref-score",
+ "data_set_field": "ref-score"
+ },
+ {
+ "transform_field": "cmp-id",
+ "data_set_field": "cmp-id"
+ },
+ {
+ "transform_field": "cmp-name",
+ "data_set_field": "cmp-name"
+ },
+ {
+ "transform_field": "cmp-score",
+ "data_set_field": "cmp-score"
+ }
+ ],
+ "field_order": [
+ "id"
+ ],
+ "data_set_name": "golden-merge-rows-passthrough",
+ "transform_name": "validate"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0077-merge-rows-passthrough UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0077-merge-rows-passthrough.hpl"
+}
\ No newline at end of file
diff --git
a/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRows.java
b/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRows.java
index 0d2a10aa3b..9776f959de 100644
---
a/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRows.java
+++
b/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRows.java
@@ -17,12 +17,14 @@
package org.apache.hop.pipeline.transforms.mergerows;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopRowException;
import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.exception.HopValueException;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowDataUtil;
@@ -115,6 +117,43 @@ public class MergeRows extends
BaseTransform<MergeRowsMeta, MergeRowsData> {
}
}
}
+
+ // Calculate the indices for the passthrough fields.
+ //
+ data.passThroughIndexes = new ArrayList<>();
+ data.oneRowMeta = data.oneRowSet.getRowMeta();
+ if (data.oneRowMeta == null) {
+ data.oneRowMeta =
+ getPipelineMeta().getPrevTransformFields(this,
meta.getReferenceTransform());
+ }
+ data.twoRowMeta = data.twoRowSet.getRowMeta();
+ if (data.twoRowMeta == null) {
+ data.twoRowMeta =
+ getPipelineMeta().getPrevTransformFields(this,
meta.getCompareTransform());
+ }
+ for (PassThroughField field : meta.getPassThroughFields()) {
+ int index;
+ if (field.isReferenceField()) {
+ index = data.oneRowMeta.indexOfValue(field.getSourceField());
+ if (index < 0) {
+ throw new HopTransformException(
+ "Unable to find passthrough field '"
+ + field.getSourceField()
+ + "' from reference transform "
+ + meta.getReferenceTransform());
+ }
+ } else {
+ index = data.twoRowMeta.indexOfValue(field.getSourceField());
+ if (index < 0) {
+ throw new HopTransformException(
+ "Unable to find passthrough field '"
+ + field.getSourceField()
+ + "' from compare transform "
+ + meta.getCompareTransform());
+ }
+ }
+ data.passThroughIndexes.add(index);
+ }
}
if (isRowLevel()) {
@@ -130,23 +169,13 @@ public class MergeRows extends
BaseTransform<MergeRowsMeta, MergeRowsData> {
if (data.outputRowMeta == null) {
data.outputRowMeta = new RowMeta();
- if (data.one != null) {
- meta.getFields(
- data.outputRowMeta,
- getTransformName(),
- new IRowMeta[] {data.oneRowSet.getRowMeta()},
- null,
- this,
- metadataProvider);
- } else {
- meta.getFields(
- data.outputRowMeta,
- getTransformName(),
- new IRowMeta[] {data.twoRowSet.getRowMeta()},
- null,
- this,
- metadataProvider);
- }
+ meta.getFields(
+ data.outputRowMeta,
+ getTransformName(),
+ new IRowMeta[] {data.oneRowMeta, data.twoRowMeta},
+ null,
+ this,
+ metadataProvider);
}
Object[] outputRow;
@@ -154,24 +183,27 @@ public class MergeRows extends
BaseTransform<MergeRowsMeta, MergeRowsData> {
String differenceJson = "{\"changes\":[]}";
boolean getDifference = StringUtils.isNotEmpty(meta.getDiffJsonField());
- if (data.one == null && data.two != null) { // Record 2 is flagged as new!
+ // Remember the rows used to compare: copy rows 'one' and 'two'.
+ copyOneTwo();
+ if (data.one == null && data.two != null) { // Record 2 is flagged as new!
outputRow = data.two;
flagField = VALUE_NEW;
// Also get a next row from compare rowset...
data.two = getRowFrom(data.twoRowSet);
+
} else if (data.one != null && data.two == null) { // Record 1 is flagged
as deleted!
outputRow = data.one;
flagField = VALUE_DELETED;
// Also get a next row from reference rowset...
data.one = getRowFrom(data.oneRowSet);
+
} else { // OK, Here is the real start of the compare code!
int compare = data.oneRowSet.getRowMeta().compare(data.one, data.two,
data.keyNrs);
if (compare == 0) { // The Key matches, we CAN compare the two rows...
-
int compareValues = 0;
if (getDifference) {
JSONObject j = new JSONObject();
@@ -214,27 +246,52 @@ public class MergeRows extends
BaseTransform<MergeRowsMeta, MergeRowsData> {
data.two = getRowFrom(data.twoRowSet);
} else {
if (compare < 0) { // one < two
-
outputRow = data.one;
flagField = VALUE_DELETED;
-
data.one = getRowFrom(data.oneRowSet);
} else {
outputRow = data.two;
flagField = VALUE_NEW;
-
data.two = getRowFrom(data.twoRowSet);
}
}
}
+ int extraPassthroughFields = meta.getPassThroughFields().size();
+
// Optionally add the difference JSON field
//
outputRow = RowDataUtil.resizeArray(outputRow, data.outputRowMeta.size());
+ int tailIndex = data.outputRowMeta.size() - 2 - extraPassthroughFields;
if (getDifference && differenceJson != null) {
- outputRow[data.outputRowMeta.size() - 2] = differenceJson;
+ outputRow[tailIndex++] = differenceJson;
+ } else {
+ tailIndex++;
+ }
+ outputRow[tailIndex++] = flagField;
+
+ // Copy the passthrough fields
+ //
+ for (int i = 0; i < extraPassthroughFields; i++) {
+ int sourceIndex = data.passThroughIndexes.get(i);
+ PassThroughField field = meta.getPassThroughFields().get(i);
+ if (field.isReferenceField()) {
+ // If the record is deleted, identical or changed we copy data
+ // from the reference row.
+ //
+ if (data.oneCopy != null && !VALUE_NEW.equals(flagField)) {
+ outputRow[tailIndex] = data.oneCopy[sourceIndex];
+ }
+ } else {
+ // If the record is new, identical or changed we copy data
+ // from the compared-to row.
+ //
+ if (data.twoCopy != null && !VALUE_DELETED.equals(flagField)) {
+ outputRow[tailIndex] = data.twoCopy[sourceIndex];
+ }
+ }
+ tailIndex++;
}
- outputRow[data.outputRowMeta.size() - 1] = flagField;
// send the row to the next transforms...
putRow(data.outputRowMeta, outputRow);
@@ -246,6 +303,23 @@ public class MergeRows extends
BaseTransform<MergeRowsMeta, MergeRowsData> {
return true;
}
+ private void copyOneTwo() throws HopValueException {
+ data.oneCopy = null;
+ data.twoCopy = null;
+
+ // We only need the exact current record values in case we want to pass
through fields
+ //
+ if (meta.getPassThroughFields().isEmpty()) {
+ return;
+ }
+ if (data.one != null) {
+ data.oneCopy = data.oneRowMeta.cloneRow(data.one);
+ }
+ if (data.two != null) {
+ data.twoCopy = data.twoRowMeta.cloneRow(data.two);
+ }
+ }
+
/**
* @see ITransform#init
*/
diff --git
a/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsData.java
b/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsData.java
index 3f2cefc3c8..b65426267d 100644
---
a/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsData.java
+++
b/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsData.java
@@ -17,6 +17,7 @@
package org.apache.hop.pipeline.transforms.mergerows;
+import java.util.List;
import org.apache.hop.core.IRowSet;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.pipeline.transform.BaseTransformData;
@@ -32,7 +33,13 @@ public class MergeRowsData extends BaseTransformData
implements ITransformData {
public int[] valueNrs;
public IRowSet oneRowSet;
+ public IRowMeta oneRowMeta;
public IRowSet twoRowSet;
+ public IRowMeta twoRowMeta;
+
+ public List<Integer> passThroughIndexes;
+ public Object[] oneCopy;
+ public Object[] twoCopy;
public MergeRowsData() {
super();
diff --git
a/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsDialog.java
b/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsDialog.java
index 4cd8e42908..6bcf8975d1 100644
---
a/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsDialog.java
+++
b/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsDialog.java
@@ -19,8 +19,10 @@ package org.apache.hop.pipeline.transforms.mergerows;
import java.util.List;
import org.apache.hop.core.Const;
+import org.apache.hop.core.Props;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
@@ -31,17 +33,19 @@ import org.apache.hop.ui.core.PropsUi;
import org.apache.hop.ui.core.dialog.BaseDialog;
import org.apache.hop.ui.core.dialog.ErrorDialog;
import org.apache.hop.ui.core.dialog.MessageDialogWithToggle;
+import org.apache.hop.ui.core.gui.GuiResource;
import org.apache.hop.ui.core.widget.ColumnInfo;
import org.apache.hop.ui.core.widget.TableView;
import org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.CCombo;
+import org.eclipse.swt.custom.CTabFolder;
+import org.eclipse.swt.custom.CTabItem;
import org.eclipse.swt.events.ModifyListener;
-import org.eclipse.swt.events.SelectionAdapter;
-import org.eclipse.swt.events.SelectionEvent;
import org.eclipse.swt.layout.FormAttachment;
import org.eclipse.swt.layout.FormData;
import org.eclipse.swt.widgets.Button;
+import org.eclipse.swt.widgets.Composite;
import org.eclipse.swt.widgets.Label;
import org.eclipse.swt.widgets.Shell;
import org.eclipse.swt.widgets.TableItem;
@@ -51,16 +55,19 @@ public class MergeRowsDialog extends BaseTransformDialog {
private static final Class<?> PKG = MergeRowsMeta.class;
public static final String STRING_SORT_WARNING_PARAMETER =
"MergeRowsSortWarning";
- private CCombo wReference;
+ private static final String YES = BaseMessages.getString("System.Combo.Yes");
+ private static final String NO = BaseMessages.getString("System.Combo.No");
- private CCombo wCompare;
+ private CTabFolder wTabFolder;
+ private CCombo wReference;
+ private CCombo wCompare;
private Text wFlagField;
private Text wDiffField;
private TableView wKeys;
-
private TableView wValues;
+ private TableView wExtra;
private final MergeRowsMeta input;
@@ -79,43 +86,46 @@ public class MergeRowsDialog extends BaseTransformDialog {
ModifyListener lsMod = e -> input.setChanged();
backupChanged = input.hasChanged();
- Button wbKeys = new Button(shell, SWT.PUSH);
- wbKeys.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.KeyFields.Button"));
- FormData fdbKeys = new FormData();
- fdbKeys.bottom = new FormAttachment(wOk, -margin);
- fdbKeys.left = new FormAttachment(0, 0);
- fdbKeys.right = new FormAttachment(50, -margin);
- wbKeys.setLayoutData(fdbKeys);
- wbKeys.addSelectionListener(
- new SelectionAdapter() {
+ // We want 4 tabs: Sources, Keys, Values, Extra
+ //
+ wTabFolder = new CTabFolder(shell, SWT.BORDER);
+ PropsUi.setLook(wTabFolder, Props.WIDGET_STYLE_TAB);
- @Override
- public void widgetSelected(SelectionEvent e) {
- getKeys();
- }
- });
+ addSourcesTab();
+ addKeysTab();
+ addValuesTab();
+ addExtraTab();
- Button wbValues = new Button(shell, SWT.PUSH);
- wbValues.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.ValueFields.Button"));
- FormData fdbValues = new FormData();
- fdbValues.bottom = new FormAttachment(wOk, -margin);
- fdbValues.left = new FormAttachment(50, 0);
- fdbValues.right = new FormAttachment(100, 0);
- wbValues.setLayoutData(fdbValues);
- wbValues.addSelectionListener(
- new SelectionAdapter() {
+ FormData fdTabFolder = new FormData();
+ fdTabFolder.left = new FormAttachment(0, 0);
+ fdTabFolder.right = new FormAttachment(100, 0);
+ fdTabFolder.top = new FormAttachment(wTransformName, margin);
+ fdTabFolder.bottom = new FormAttachment(wOk, -2 * margin);
+ wTabFolder.setLayoutData(fdTabFolder);
+ wTabFolder.setSelection(0);
- @Override
- public void widgetSelected(SelectionEvent e) {
- getValues();
- }
- });
+ getData();
+ input.setChanged(backupChanged);
+ focusTransformName();
+ BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel());
+
+ return transformName;
+ }
+
+ private void addSourcesTab() {
+ CTabItem wSourcesTab = new CTabItem(wTabFolder, SWT.NONE);
+ wSourcesTab.setFont(GuiResource.getInstance().getFontDefault());
+ wSourcesTab.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.SourcesTab.CTabItem"));
+
+ Composite wSourcesComp = new Composite(wTabFolder, SWT.NONE);
+ PropsUi.setLook(wSourcesComp);
+ wSourcesComp.setLayout(props.createFormLayout());
// Get the previous transforms...
String[] previousTransforms =
pipelineMeta.getPrevTransformNames(transformName);
// Send 'True' data to...
- Label wlReference = new Label(shell, SWT.RIGHT);
+ Label wlReference = new Label(wSourcesComp, SWT.RIGHT);
wlReference.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.Reference.Label"));
PropsUi.setLook(wlReference);
FormData fdlReference = new FormData();
@@ -123,13 +133,11 @@ public class MergeRowsDialog extends BaseTransformDialog {
fdlReference.right = new FormAttachment(middle, -margin);
fdlReference.top = new FormAttachment(wSpacer, margin);
wlReference.setLayoutData(fdlReference);
- wReference = new CCombo(shell, SWT.BORDER);
+ wReference = new CCombo(wSourcesComp, SWT.BORDER);
PropsUi.setLook(wReference);
-
if (previousTransforms != null) {
wReference.setItems(previousTransforms);
}
-
wReference.addModifyListener(lsMod);
FormData fdReference = new FormData();
fdReference.left = new FormAttachment(middle, 0);
@@ -138,7 +146,7 @@ public class MergeRowsDialog extends BaseTransformDialog {
wReference.setLayoutData(fdReference);
// Send 'False' data to...
- Label wlCompare = new Label(shell, SWT.RIGHT);
+ Label wlCompare = new Label(wSourcesComp, SWT.RIGHT);
wlCompare.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.Compare.Label"));
PropsUi.setLook(wlCompare);
FormData fdlCompare = new FormData();
@@ -146,13 +154,11 @@ public class MergeRowsDialog extends BaseTransformDialog {
fdlCompare.right = new FormAttachment(middle, -margin);
fdlCompare.top = new FormAttachment(wReference, margin);
wlCompare.setLayoutData(fdlCompare);
- wCompare = new CCombo(shell, SWT.BORDER);
+ wCompare = new CCombo(wSourcesComp, SWT.BORDER);
PropsUi.setLook(wCompare);
-
if (previousTransforms != null) {
wCompare.setItems(previousTransforms);
}
-
wCompare.addModifyListener(lsMod);
FormData fdCompare = new FormData();
fdCompare.top = new FormAttachment(wReference, margin);
@@ -161,52 +167,80 @@ public class MergeRowsDialog extends BaseTransformDialog {
wCompare.setLayoutData(fdCompare);
// The flag field line
- Label wlFlagField = new Label(shell, SWT.RIGHT);
+ Label wlFlagField = new Label(wSourcesComp, SWT.RIGHT);
wlFlagField.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.FlagField.Label"));
PropsUi.setLook(wlFlagField);
- FormData fdlFlagfield = new FormData();
- fdlFlagfield.left = new FormAttachment(0, 0);
- fdlFlagfield.right = new FormAttachment(middle, -margin);
- fdlFlagfield.top = new FormAttachment(wCompare, margin);
- wlFlagField.setLayoutData(fdlFlagfield);
- wFlagField = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ FormData fdlFlagField = new FormData();
+ fdlFlagField.left = new FormAttachment(0, 0);
+ fdlFlagField.right = new FormAttachment(middle, -margin);
+ fdlFlagField.top = new FormAttachment(wCompare, margin);
+ wlFlagField.setLayoutData(fdlFlagField);
+ wFlagField = new Text(wSourcesComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
PropsUi.setLook(wFlagField);
wFlagField.addModifyListener(lsMod);
- FormData fdFlagfield = new FormData();
- fdFlagfield.top = new FormAttachment(wCompare, margin);
- fdFlagfield.left = new FormAttachment(middle, 0);
- fdFlagfield.right = new FormAttachment(100, 0);
- wFlagField.setLayoutData(fdFlagfield);
+ FormData fdFlagField = new FormData();
+ fdFlagField.top = new FormAttachment(wCompare, margin);
+ fdFlagField.left = new FormAttachment(middle, 0);
+ fdFlagField.right = new FormAttachment(100, 0);
+ wFlagField.setLayoutData(fdFlagField);
// The flag field line
- Label wlDiffField = new Label(shell, SWT.RIGHT);
+ Label wlDiffField = new Label(wSourcesComp, SWT.RIGHT);
wlDiffField.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.DiffField.Label"));
PropsUi.setLook(wlDiffField);
- FormData fdlDifffield = new FormData();
- fdlDifffield.left = new FormAttachment(0, 0);
- fdlDifffield.right = new FormAttachment(middle, -margin);
- fdlDifffield.top = new FormAttachment(wFlagField, margin);
- wlDiffField.setLayoutData(fdlDifffield);
- wDiffField = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ FormData fdlDiffField = new FormData();
+ fdlDiffField.left = new FormAttachment(0, 0);
+ fdlDiffField.right = new FormAttachment(middle, -margin);
+ fdlDiffField.top = new FormAttachment(wFlagField, margin);
+ wlDiffField.setLayoutData(fdlDiffField);
+ wDiffField = new Text(wSourcesComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
PropsUi.setLook(wDiffField);
wDiffField.addModifyListener(lsMod);
- FormData fdDifffield = new FormData();
- fdDifffield.top = new FormAttachment(wFlagField, margin);
- fdDifffield.left = new FormAttachment(middle, 0);
- fdDifffield.right = new FormAttachment(100, 0);
- wDiffField.setLayoutData(fdDifffield);
+ FormData fdDiffField = new FormData();
+ fdDiffField.top = new FormAttachment(wFlagField, margin);
+ fdDiffField.left = new FormAttachment(middle, 0);
+ fdDiffField.right = new FormAttachment(100, 0);
+ wDiffField.setLayoutData(fdDiffField);
+
+ FormData fdSourcesComp = new FormData();
+ fdSourcesComp.left = new FormAttachment(0, 0);
+ fdSourcesComp.top = new FormAttachment(0, 0);
+ fdSourcesComp.right = new FormAttachment(100, 0);
+ fdSourcesComp.bottom = new FormAttachment(100, 0);
+ wSourcesComp.setLayoutData(fdSourcesComp);
+
+ wSourcesComp.layout();
+ wSourcesTab.setControl(wSourcesComp);
+ }
+
+ private void addKeysTab() {
+ CTabItem wKeysTab = new CTabItem(wTabFolder, SWT.NONE);
+ wKeysTab.setFont(GuiResource.getInstance().getFontDefault());
+ wKeysTab.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.KeysTab.CTabItem"));
+
+ Composite wKeysComp = new Composite(wTabFolder, SWT.NONE);
+ PropsUi.setLook(wKeysComp);
+ wKeysComp.setLayout(props.createFormLayout());
+
+ // The [Get key fields] button at the bottom:
+ Button wbKeys = new Button(wKeysComp, SWT.PUSH);
+ wbKeys.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.KeyFields.Button"));
+ FormData fdbKeys = new FormData();
+ fdbKeys.bottom = new FormAttachment(100, -margin);
+ fdbKeys.left = new FormAttachment(0, 0);
+ wbKeys.setLayoutData(fdbKeys);
+ wbKeys.addListener(SWT.Selection, e -> getKeys());
// THE KEYS TO MATCH...
- Label wlKeys = new Label(shell, SWT.NONE);
+
+ Label wlKeys = new Label(wKeysComp, SWT.NONE);
wlKeys.setText(BaseMessages.getString(PKG, "MergeRowsDialog.Keys.Label"));
PropsUi.setLook(wlKeys);
FormData fdlKeys = new FormData();
fdlKeys.left = new FormAttachment(0, 0);
- fdlKeys.top = new FormAttachment(wDiffField, margin);
+ fdlKeys.top = new FormAttachment(0, 0);
wlKeys.setLayoutData(fdlKeys);
- int nrKeyRows = (input.getKeyFields() != null ?
input.getKeyFields().size() : 1);
-
ColumnInfo[] ciKeys =
new ColumnInfo[] {
new ColumnInfo(
@@ -218,10 +252,10 @@ public class MergeRowsDialog extends BaseTransformDialog {
wKeys =
new TableView(
variables,
- shell,
+ wKeysComp,
SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI | SWT.V_SCROLL |
SWT.H_SCROLL,
ciKeys,
- nrKeyRows,
+ 1,
lsMod,
props);
@@ -229,20 +263,46 @@ public class MergeRowsDialog extends BaseTransformDialog {
fdKeys.top = new FormAttachment(wlKeys, margin);
fdKeys.left = new FormAttachment(0, 0);
fdKeys.bottom = new FormAttachment(wbKeys, -margin);
- fdKeys.right = new FormAttachment(50, -margin);
+ fdKeys.right = new FormAttachment(100, 0);
wKeys.setLayoutData(fdKeys);
+ FormData fdKeysComp = new FormData();
+ fdKeysComp.left = new FormAttachment(0, 0);
+ fdKeysComp.top = new FormAttachment(0, 0);
+ fdKeysComp.right = new FormAttachment(100, 0);
+ fdKeysComp.bottom = new FormAttachment(100, 0);
+ wKeysComp.setLayoutData(fdKeysComp);
+
+ wKeysComp.layout();
+ wKeysTab.setControl(wKeysComp);
+ }
+
+ private void addValuesTab() {
+ CTabItem wValuesTab = new CTabItem(wTabFolder, SWT.NONE);
+ wValuesTab.setFont(GuiResource.getInstance().getFontDefault());
+ wValuesTab.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.ValuesTab.CTabItem"));
+
+ Composite wValuesComp = new Composite(wTabFolder, SWT.NONE);
+ PropsUi.setLook(wValuesComp);
+ wValuesComp.setLayout(props.createFormLayout());
+
+ Button wbValues = new Button(wValuesComp, SWT.PUSH);
+ wbValues.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.ValueFields.Button"));
+ FormData fdbValues = new FormData();
+ fdbValues.bottom = new FormAttachment(100, -margin);
+ fdbValues.left = new FormAttachment(0, 0);
+ wbValues.setLayoutData(fdbValues);
+ wbValues.addListener(SWT.Selection, e -> getValues());
+
// VALUES TO COMPARE
- Label wlValues = new Label(shell, SWT.NONE);
+ Label wlValues = new Label(wValuesComp, SWT.NONE);
wlValues.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.Values.Label"));
PropsUi.setLook(wlValues);
FormData fdlValues = new FormData();
- fdlValues.left = new FormAttachment(50, 0);
+ fdlValues.left = new FormAttachment(0, 0);
fdlValues.top = new FormAttachment(wDiffField, margin);
wlValues.setLayoutData(fdlValues);
- int nrValueRows = (input.getValueFields() != null ?
input.getValueFields().size() : 1);
-
ColumnInfo[] ciValues =
new ColumnInfo[] {
new ColumnInfo(
@@ -254,26 +314,129 @@ public class MergeRowsDialog extends BaseTransformDialog
{
wValues =
new TableView(
variables,
- shell,
+ wValuesComp,
SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI | SWT.V_SCROLL |
SWT.H_SCROLL,
ciValues,
- nrValueRows,
+ 1,
lsMod,
props);
FormData fdValues = new FormData();
fdValues.top = new FormAttachment(wlValues, margin);
- fdValues.left = new FormAttachment(50, 0);
+ fdValues.left = new FormAttachment(0, 0);
fdValues.bottom = new FormAttachment(wbValues, -margin);
fdValues.right = new FormAttachment(100, 0);
wValues.setLayoutData(fdValues);
- getData();
- input.setChanged(backupChanged);
- focusTransformName();
- BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel());
+ FormData fdValuesComp = new FormData();
+ fdValuesComp.left = new FormAttachment(0, 0);
+ fdValuesComp.top = new FormAttachment(0, 0);
+ fdValuesComp.right = new FormAttachment(100, 0);
+ fdValuesComp.bottom = new FormAttachment(100, 0);
+ wValuesComp.setLayoutData(fdValuesComp);
- return transformName;
+ wValuesComp.layout();
+ wValuesTab.setControl(wValuesComp);
+ }
+
+ private void addExtraTab() {
+ CTabItem wExtraTab = new CTabItem(wTabFolder, SWT.NONE);
+ wExtraTab.setFont(GuiResource.getInstance().getFontDefault());
+ wExtraTab.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.ExtraTab.CTabItem"));
+
+ Composite wExtraComp = new Composite(wTabFolder, SWT.NONE);
+ PropsUi.setLook(wExtraComp);
+ wExtraComp.setLayout(props.createFormLayout());
+
+ // The [Get fields] button at the bottom
+ Button wbExtra = new Button(wExtraComp, SWT.PUSH);
+ wbExtra.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.ExtraFields.Button"));
+ FormData fdbExtra = new FormData();
+ fdbExtra.bottom = new FormAttachment(100, -margin);
+ fdbExtra.left = new FormAttachment(0, 0);
+ wbExtra.setLayoutData(fdbExtra);
+ wbExtra.addListener(SWT.Selection, e -> getPassThroughFields());
+
+ // Extra fields to pass through to the output
+ Label wlExtra = new Label(wExtraComp, SWT.NONE);
+ wlExtra.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.Extra.Label"));
+ PropsUi.setLook(wlExtra);
+ FormData fdlExtra = new FormData();
+ fdlExtra.left = new FormAttachment(0, 0);
+ fdlExtra.top = new FormAttachment(wDiffField, margin);
+ wlExtra.setLayoutData(fdlExtra);
+
+ ColumnInfo[] ciExtra =
+ new ColumnInfo[] {
+ new ColumnInfo(
+ BaseMessages.getString(PKG,
"MergeRowsDialog.ExtraColumn.Reference.Label"),
+ ColumnInfo.COLUMN_TYPE_CCOMBO,
+ YES,
+ NO),
+ new ColumnInfo(
+ BaseMessages.getString(PKG,
"MergeRowsDialog.ExtraColumn.Field.Label"),
+ ColumnInfo.COLUMN_TYPE_TEXT,
+ false),
+ new ColumnInfo(
+ BaseMessages.getString(PKG,
"MergeRowsDialog.ExtraColumn.RenameTo.Label"),
+ ColumnInfo.COLUMN_TYPE_TEXT,
+ false),
+ };
+
+ wExtra =
+ new TableView(
+ variables,
+ wExtraComp,
+ SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI | SWT.V_SCROLL |
SWT.H_SCROLL,
+ ciExtra,
+ 1,
+ lsMod,
+ props);
+
+ FormData fdExtra = new FormData();
+ fdExtra.top = new FormAttachment(wlExtra, margin);
+ fdExtra.left = new FormAttachment(0, 0);
+ fdExtra.bottom = new FormAttachment(wbExtra, -margin);
+ fdExtra.right = new FormAttachment(100, 0);
+ wExtra.setLayoutData(fdExtra);
+
+ FormData fdExtraComp = new FormData();
+ fdExtraComp.left = new FormAttachment(0, 0);
+ fdExtraComp.top = new FormAttachment(0, 0);
+ fdExtraComp.right = new FormAttachment(100, 0);
+ fdExtraComp.bottom = new FormAttachment(100, 0);
+ wExtraComp.setLayoutData(fdExtraComp);
+
+ wExtraComp.layout();
+ wExtraTab.setControl(wExtraComp);
+ }
+
+ private void getPassThroughFields() {
+ MergeRowsMeta meta = new MergeRowsMeta();
+ getInfo(meta);
+
+ // Get all fields from both reference and compare transforms
+ // and add them to the wExtra table view.
+ try {
+ IRowMeta refRowMeta =
+ pipelineMeta.getTransformFields(variables,
meta.getReferenceTransform());
+ for (IValueMeta refValueMeta : refRowMeta.getValueMetaList()) {
+ TableItem item = new TableItem(wExtra.table, SWT.NONE);
+ item.setText(1, YES);
+ item.setText(2, refValueMeta.getName());
+ item.setText(3, "ref-" + refValueMeta.getName());
+ }
+ IRowMeta cmpRowMeta = pipelineMeta.getTransformFields(variables,
meta.getCompareTransform());
+ for (IValueMeta cmpValueMeta : cmpRowMeta.getValueMetaList()) {
+ TableItem item = new TableItem(wExtra.table, SWT.NONE);
+ item.setText(1, NO);
+ item.setText(2, cmpValueMeta.getName());
+ item.setText(3, "cmp-" + cmpValueMeta.getName());
+ }
+ wExtra.optimizeTableView();
+ } catch (Exception e) {
+ new ErrorDialog(shell, "Error", "Error adding passthrough fields", e);
+ }
}
/** Copy information from the meta-data input to the dialog fields. */
@@ -286,13 +449,23 @@ public class MergeRowsDialog extends BaseTransformDialog {
wDiffField.setText(Const.NVL(input.getDiffJsonField(), ""));
for (int i = 0; i < input.getKeyFields().size(); i++) {
- TableItem item = wKeys.table.getItem(i);
+ TableItem item = new TableItem(wKeys.table, SWT.NONE);
item.setText(1, Const.NVL(input.getKeyFields().get(i), ""));
}
+ wKeys.optimizeTableView();
for (int i = 0; i < input.getValueFields().size(); i++) {
- TableItem item = wValues.table.getItem(i);
+ TableItem item = new TableItem(wValues.table, SWT.NONE);
item.setText(1, Const.NVL(input.getValueFields().get(i), ""));
}
+ wValues.optimizeTableView();
+ for (int i = 0; i < input.getPassThroughFields().size(); i++) {
+ PassThroughField field = input.getPassThroughFields().get(i);
+ TableItem item = new TableItem(wExtra.table, SWT.NONE);
+ item.setText(1, field.isReferenceField() ? YES : NO);
+ item.setText(2, Const.NVL(field.getSourceField(), ""));
+ item.setText(3, Const.NVL(field.getRenameTo(), ""));
+ }
+ wExtra.optimizeTableView();
}
private void cancel() {
@@ -305,33 +478,11 @@ public class MergeRowsDialog extends BaseTransformDialog {
if (Utils.isEmpty(wTransformName.getText())) {
return;
}
-
- input.setReferenceTransform(wReference.getText());
- input.setCompareTransform(wCompare.getText());
- List<IStream> infoStreams = input.getTransformIOMeta().getInfoStreams();
-
infoStreams.get(0).setTransformMeta(pipelineMeta.findTransform(wReference.getText()));
-
infoStreams.get(1).setTransformMeta(pipelineMeta.findTransform(wCompare.getText()));
- input.setFlagField(wFlagField.getText());
- input.setDiffJsonField(wDiffField.getText());
-
- int nrKeys = wKeys.nrNonEmpty();
- int nrValues = wValues.nrNonEmpty();
-
- input.getKeyFields().clear();
- for (int i = 0; i < nrKeys; i++) {
- TableItem item = wKeys.getNonEmpty(i);
- input.getKeyFields().add(item.getText(1));
- }
-
- input.getValueFields().clear();
- for (int i = 0; i < nrValues; i++) {
- TableItem item = wValues.getNonEmpty(i);
- input.getValueFields().add(item.getText(1));
- }
+ getInfo(input);
transformName = wTransformName.getText(); // return value
- if (nrKeys > 0
+ if (!input.getKeyFields().isEmpty()
&&
"Y".equalsIgnoreCase(props.getCustomParameter(STRING_SORT_WARNING_PARAMETER,
"Y"))) {
MessageDialogWithToggle md =
new MessageDialogWithToggle(
@@ -353,6 +504,34 @@ public class MergeRowsDialog extends BaseTransformDialog {
dispose();
}
+ private void getInfo(MergeRowsMeta meta) {
+ meta.setReferenceTransform(wReference.getText());
+ meta.setCompareTransform(wCompare.getText());
+ List<IStream> infoStreams = meta.getTransformIOMeta().getInfoStreams();
+
infoStreams.get(0).setTransformMeta(pipelineMeta.findTransform(wReference.getText()));
+
infoStreams.get(1).setTransformMeta(pipelineMeta.findTransform(wCompare.getText()));
+ meta.setFlagField(wFlagField.getText());
+ meta.setDiffJsonField(wDiffField.getText());
+
+ meta.getKeyFields().clear();
+ for (TableItem item : wKeys.getNonEmptyItems()) {
+ meta.getKeyFields().add(item.getText(1));
+ }
+
+ meta.getValueFields().clear();
+ for (TableItem item : wValues.getNonEmptyItems()) {
+ meta.getValueFields().add(item.getText(1));
+ }
+
+ meta.getPassThroughFields().clear();
+ for (TableItem item : wExtra.getNonEmptyItems()) {
+ boolean reference = YES.equalsIgnoreCase(item.getText(1));
+ String fieldName = item.getText(2);
+ String renameTo = item.getText(3);
+ meta.getPassThroughFields().add(new PassThroughField(fieldName,
renameTo, reference));
+ }
+ }
+
private void getKeys() {
try {
TransformMeta transformMeta =
pipelineMeta.findTransform(wReference.getText());
diff --git
a/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsMeta.java
b/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsMeta.java
index c72bff7080..1f1f31ff3d 100644
---
a/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsMeta.java
+++
b/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsMeta.java
@@ -99,18 +99,30 @@ public class MergeRowsMeta extends
BaseTransformMeta<MergeRows, MergeRowsData> {
injectionKeyDescription = "MergeRows.Injection.DIFF_FIELD")
private String diffJsonField;
+ @HopMetadataProperty(
+ groupKey = "passthrough-fields",
+ key = "passthrough-field",
+ injectionGroupKey = "PASSTHROUGH_FIELDS",
+ injectionGroupDescription = "MergeRows.Injection.PASSTHROUGH_FIELDS",
+ injectionKey = "PASSTHROUGH_FIELD",
+ injectionKeyDescription = "MergeRows.Injection.PASSTHROUGH_FIELD")
+ private List<PassThroughField> passThroughFields;
+
public MergeRowsMeta() {
super();
keyFields = new ArrayList<>();
valueFields = new ArrayList<>();
+ passThroughFields = new ArrayList<>();
}
public MergeRowsMeta(MergeRowsMeta m) {
+ this();
this.flagField = m.flagField;
this.keyFields = new ArrayList<>(m.keyFields);
this.valueFields = new ArrayList<>(m.valueFields);
this.referenceTransform = m.referenceTransform;
this.compareTransform = m.compareTransform;
+ m.getPassThroughFields().forEach(f -> this.passThroughFields.add(new
PassThroughField(f)));
}
@Override
@@ -165,6 +177,41 @@ public class MergeRowsMeta extends
BaseTransformMeta<MergeRows, MergeRowsData> {
IValueMeta flagFieldValue = new ValueMetaString(flagField);
flagFieldValue.setOrigin(name);
r.addValueMeta(flagFieldValue);
+
+ if (info == null || info.length != 2) {
+ return;
+ }
+
+ // Add the passthrough fields
+ //
+ for (PassThroughField field : passThroughFields) {
+ IValueMeta valueMeta = null;
+ if (field.isReferenceField()) {
+ if (info[0] != null) {
+ valueMeta = info[0].searchValueMeta(field.getSourceField());
+ if (valueMeta == null) {
+ throw new HopTransformException(
+ "Unable to find passthrough reference field '" +
field.getSourceField() + "'");
+ }
+ }
+ } else {
+ if (info[1] != null) {
+ valueMeta = info[1].searchValueMeta(field.getSourceField());
+ if (valueMeta == null) {
+ throw new HopTransformException(
+ "Unable to find passthrough compare field '" +
field.getSourceField() + "'");
+ }
+ }
+ }
+ if (valueMeta != null) {
+ // create a copy to prevent renaming fields from the reference/compare
transforms.
+ valueMeta = valueMeta.clone();
+ if (StringUtils.isNotEmpty(field.getRenameTo())) {
+ valueMeta.setName(field.getRenameTo());
+ }
+ r.addValueMeta(valueMeta);
+ }
+ }
}
@Override
diff --git
a/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsData.java
b/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/PassThroughField.java
similarity index 52%
copy from
plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsData.java
copy to
plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/PassThroughField.java
index 3f2cefc3c8..03cc7dde70 100644
---
a/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsData.java
+++
b/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/PassThroughField.java
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,24 +17,30 @@
package org.apache.hop.pipeline.transforms.mergerows;
-import org.apache.hop.core.IRowSet;
-import org.apache.hop.core.row.IRowMeta;
-import org.apache.hop.pipeline.transform.BaseTransformData;
-import org.apache.hop.pipeline.transform.ITransformData;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.metadata.api.HopMetadataProperty;
-@SuppressWarnings("java:S1104")
-public class MergeRowsData extends BaseTransformData implements ITransformData
{
- public IRowMeta outputRowMeta;
+@Getter
+@Setter
+public class PassThroughField {
+ @HopMetadataProperty private String sourceField;
- public Object[] one;
- public Object[] two;
- public int[] keyNrs;
- public int[] valueNrs;
+ @HopMetadataProperty private String renameTo;
- public IRowSet oneRowSet;
- public IRowSet twoRowSet;
+ @HopMetadataProperty private boolean referenceField;
- public MergeRowsData() {
- super();
+ public PassThroughField() {}
+
+ public PassThroughField(String sourceField, String renameTo, boolean
referenceField) {
+ this.sourceField = sourceField;
+ this.renameTo = renameTo;
+ this.referenceField = referenceField;
+ }
+
+ public PassThroughField(PassThroughField f) {
+ this.sourceField = f.sourceField;
+ this.renameTo = f.renameTo;
+ this.referenceField = f.referenceField;
}
}
diff --git
a/plugins/transforms/mergerows/src/main/resources/org/apache/hop/pipeline/transforms/mergerows/messages/messages_en_US.properties
b/plugins/transforms/mergerows/src/main/resources/org/apache/hop/pipeline/transforms/mergerows/messages/messages_en_US.properties
index 566d8c9e3f..8adc64c5fc 100644
---
a/plugins/transforms/mergerows/src/main/resources/org/apache/hop/pipeline/transforms/mergerows/messages/messages_en_US.properties
+++
b/plugins/transforms/mergerows/src/main/resources/org/apache/hop/pipeline/transforms/mergerows/messages/messages_en_US.properties
@@ -24,6 +24,8 @@ MergeRows.Injection.KEY_FIELDS=Specify key fields to compare.
MergeRows.Injection.KEY_FIELD=The key field to match with
MergeRows.Injection.VALUE_FIELDS=Specify value fields to compare.
MergeRows.Injection.VALUE_FIELD=The field to compare
+MergeRows.Injection.PASSTHROUGH_FIELDS = The fields to pass through to the
result
+MergeRows.Injection.PASSTHROUGH_FIELD = One fields to pass through to the
result
MergeRows.LineNumber=linenr
MergeRows.Log.BothTrueAndFalseNeeded=Both the ''true'' and the ''false''
transforms need to be supplied, or neither
MergeRows.Log.DataInfo=ONE\: {0} / TWO\:
@@ -56,4 +58,13 @@ MergeRowsMeta.Exception.FlagFieldNotSpecified=The flag field
is not specified.
MergeRowsMeta.Exception.UnableToLoadTransformMeta=Unable to load transform
info from XML
MergeRowsMeta.InfoStream.FirstStream.Description=Reference stream to merge
MergeRowsMeta.InfoStream.SecondStream.Description=Compare (changed data)
stream to merge
-MergeRowsMeta.keyword=merge,row
+MergeRowsMeta.keyword=merge,row,difference
+MergeRowsDialog.ExtraFields.Button = Get fields to pass through
+MergeRowsDialog.Extra.Label=Extra fields to pass through from sources to the
output\:
+MergeRowsDialog.ExtraColumn.Reference.Label=Reference?
+MergeRowsDialog.ExtraColumn.Field.Label=Source field
+MergeRowsDialog.ExtraColumn.RenameTo.Label=Rename to
+MergeRowsDialog.SourcesTab.CTabItem = Sources
+MergeRowsDialog.KeysTab.CTabItem = Keys
+MergeRowsDialog.ValuesTab.CTabItem = Values
+MergeRowsDialog.ExtraTab.CTabItem = Extra
