This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 81bde2d4e7 issue #7288 : add prefix, separator and suffix options to
transform Add Checksum (#7289)
81bde2d4e7 is described below
commit 81bde2d4e79fde12c9872d47b4aeb41020d05b01
Author: Matt Casters <[email protected]>
AuthorDate: Mon Jun 15 12:46:50 2026 +0200
issue #7288 : add prefix, separator and suffix options to transform Add
Checksum (#7289)
---
.../pages/pipeline/transforms/addchecksum.adoc | 6 +
.../transforms/0043-checksum-prefix.hpl | 460 +++++++++++++++++++++
.../transforms/main-0043-checksum.hwf | 72 +++-
.../hop/pipeline/transforms/checksum/CheckSum.java | 94 +++--
.../pipeline/transforms/checksum/CheckSumData.java | 6 +
.../transforms/checksum/CheckSumDialog.java | 66 ++-
.../pipeline/transforms/checksum/CheckSumMeta.java | 69 ++++
.../checksum/messages/messages_en_US.properties | 6 +
.../checksum/CheckSumMetaInjectionTest.java | 3 +
.../transforms/checksum/CheckSumMetaTest.java | 9 +-
.../pipeline/transforms/checksum/CheckSumTest.java | 60 ++-
11 files changed, 780 insertions(+), 71 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/addchecksum.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/addchecksum.adoc
index 3b0f5dbd41..4ec4389eea 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/addchecksum.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/addchecksum.adoc
@@ -55,6 +55,12 @@ These are the types that are available:
* SHA-256 : Secure Hash Algorithm 2 : http://en.wikipedia.org/wiki/SHA-256
|Result Type|Some checksum types allow to set different result types: String,
Hexadecimal and Binary
|Result field|The name of the result field containing the checksum
+|Prefix|An optional string to prepend before the field values when building
the input for the checksum calculation.
+This is useful to distinguish hash keys from different topics, for example
`CUST` for customer keys and `PROD` for product keys.
+|Separator|An optional string to insert between the field values when building
the input for the checksum calculation.
+This helps generate more unique hash codes when multiple fields are
concatenated.
+|Suffix|An optional string to append after the field values when building the
input for the checksum calculation.
+Like the prefix, this can be used to distinguish hash keys from different
topics.
|Fields used in the checksum|The names of the fields to include in the
checksum calculation.
Note: You can use the "Get Fields" button to insert all input fields from
previous transforms.
|===
diff --git a/integration-tests/transforms/0043-checksum-prefix.hpl
b/integration-tests/transforms/0043-checksum-prefix.hpl
new file mode 100644
index 0000000000..c8c77127d8
--- /dev/null
+++ b/integration-tests/transforms/0043-checksum-prefix.hpl
@@ -0,0 +1,460 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <pipeline_version/>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters/>
+ <name>0043-checksum</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <created_user>-</created_user>
+ <modified_user>-</modified_user>
+ <created_date>2022/04/29 07:47:12.357</created_date>
+ <modified_date>2022/04/29 07:47:12.357</modified_date>
+ </info>
+ <transform>
+ <type>DataGrid</type>
+ <name>Data grid</name>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <group/>
+ <name>product_id</name>
+ <type>Integer</type>
+ <format/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <group/>
+ <name>customer_id</name>
+ <type>Integer</type>
+ <format/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>123</item>
+ <item>123</item>
+ </line>
+ <line>
+ <item>234</item>
+ <item>234</item>
+ </line>
+ <line>
+ <item>345</item>
+ <item>345</item>
+ </line>
+ <line>
+ <item>456</item>
+ <item>456</item>
+ </line>
+ <line>
+ <item>567</item>
+ <item>567</item>
+ </line>
+ </data>
+ <distribute>N</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>48</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>CheckSum</type>
+ <name>customer_hk</name>
+ <fields>
+ <field>
+ <name>customer_id</name>
+ </field>
+ </fields>
+ <resultfieldName>customer_hk</resultfieldName>
+ <checksumtype>MD5</checksumtype>
+ <resultType>hexadecimal</resultType>
+ <separator/>
+ <prefix>CUST</prefix>
+ <suffix/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>48</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>CheckSum</type>
+ <name>product_hk</name>
+ <fields>
+ <field>
+ <name>product_id</name>
+ </field>
+ </fields>
+ <resultfieldName>product_hk</resultfieldName>
+ <checksumtype>MD5</checksumtype>
+ <resultType>hexadecimal</resultType>
+ <separator/>
+ <prefix>PROD</prefix>
+ <suffix/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>384</xloc>
+ <yloc>48</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>FilterRows</type>
+ <name>identical hash keys?</name>
+ <compare>
+ <condition>
+ <negated>N</negated>
+ <operator>-</operator>
+ <leftvalue>customer_hk</leftvalue>
+ <function>=</function>
+ <rightvalue>product_hk</rightvalue>
+ <conditions/>
+ </condition>
+ </compare>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>512</xloc>
+ <yloc>48</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>Abort</type>
+ <name>Abort</name>
+ <row_threshold>1</row_threshold>
+ <message>Hash keys with identical values and different prefix need to be
different.</message>
+ <always_log_rows>Y</always_log_rows>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>640</xloc>
+ <yloc>48</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>CheckSum</type>
+ <name>without-sep</name>
+ <fields>
+ <field>
+ <name>customer_id</name>
+ </field>
+ <field>
+ <name>product_id</name>
+ </field>
+ </fields>
+ <resultfieldName>hk</resultfieldName>
+ <checksumtype>MD5</checksumtype>
+ <resultType>hexadecimal</resultType>
+ <separator/>
+ <prefix/>
+ <suffix/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>160</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>CheckSum</type>
+ <name>with-sep</name>
+ <fields>
+ <field>
+ <name>customer_id</name>
+ </field>
+ <field>
+ <name>product_id</name>
+ </field>
+ </fields>
+ <resultfieldName>hk_sep</resultfieldName>
+ <checksumtype>MD5</checksumtype>
+ <resultType>hexadecimal</resultType>
+ <separator>||</separator>
+ <prefix/>
+ <suffix/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>384</xloc>
+ <yloc>160</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>FilterRows</type>
+ <name>identical hk?</name>
+ <compare>
+ <condition>
+ <negated>N</negated>
+ <operator>-</operator>
+ <leftvalue>hk</leftvalue>
+ <function>=</function>
+ <rightvalue>hk_sep</rightvalue>
+ <conditions/>
+ </condition>
+ </compare>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>512</xloc>
+ <yloc>160</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>Abort</type>
+ <name>Abort </name>
+ <row_threshold>1</row_threshold>
+ <message>The hash keys with separator should be different from those
without</message>
+ <always_log_rows>Y</always_log_rows>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>640</xloc>
+ <yloc>160</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>CheckSum</type>
+ <name>customer_hk </name>
+ <fields>
+ <field>
+ <name>customer_id</name>
+ </field>
+ </fields>
+ <resultfieldName>customer_hk</resultfieldName>
+ <checksumtype>MD5</checksumtype>
+ <resultType>hexadecimal</resultType>
+ <separator/>
+ <prefix/>
+ <suffix>CUST</suffix>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>272</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>CheckSum</type>
+ <name>product_hk </name>
+ <fields>
+ <field>
+ <name>product_id</name>
+ </field>
+ </fields>
+ <resultfieldName>product_hk</resultfieldName>
+ <checksumtype>MD5</checksumtype>
+ <resultType>hexadecimal</resultType>
+ <separator/>
+ <prefix/>
+ <suffix>PROD</suffix>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>384</xloc>
+ <yloc>272</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>FilterRows</type>
+ <name>identical hash keys? </name>
+ <compare>
+ <condition>
+ <negated>N</negated>
+ <operator>-</operator>
+ <leftvalue>customer_hk</leftvalue>
+ <function>=</function>
+ <rightvalue>product_hk</rightvalue>
+ <conditions/>
+ </condition>
+ </compare>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>512</xloc>
+ <yloc>272</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>Abort</type>
+ <name>Abort </name>
+ <row_threshold>1</row_threshold>
+ <message>Hash keys with identical values and different suffix need to be
different.</message>
+ <always_log_rows>Y</always_log_rows>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>640</xloc>
+ <yloc>272</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <order>
+ <hop>
+ <from>Data grid</from>
+ <to>customer_hk</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>customer_hk</from>
+ <to>product_hk</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>product_hk</from>
+ <to>identical hash keys?</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>identical hash keys?</from>
+ <to>Abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Data grid</from>
+ <to>without-sep</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>without-sep</from>
+ <to>with-sep</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>with-sep</from>
+ <to>identical hk?</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>identical hk?</from>
+ <to>Abort </to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>customer_hk </from>
+ <to>product_hk </to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>product_hk </from>
+ <to>identical hash keys? </to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>identical hash keys? </from>
+ <to>Abort </to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Data grid</from>
+ <to>customer_hk </to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <notepads/>
+ <attributes/>
+ <transform_error_handling/>
+</pipeline>
diff --git a/integration-tests/transforms/main-0043-checksum.hwf
b/integration-tests/transforms/main-0043-checksum.hwf
index 1c8e056475..ed3c8beca8 100644
--- a/integration-tests/transforms/main-0043-checksum.hwf
+++ b/integration-tests/transforms/main-0043-checksum.hwf
@@ -22,45 +22,71 @@ limitations under the License.
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
- <workflow_version/>
<created_user>-</created_user>
- <created_date>2022/04/29 10:42:16.470</created_date>
<modified_user>-</modified_user>
+ <created_date>2022/04/29 10:42:16.470</created_date>
<modified_date>2022/04/29 10:42:16.470</modified_date>
- <parameters>
- </parameters>
+ <workflow_version/>
+ <parameters/>
<actions>
<action>
- <name>Start</name>
- <description/>
- <type>SPECIAL</type>
- <attributes/>
<repeat>N</repeat>
<schedulerType>0</schedulerType>
<intervalSeconds>0</intervalSeconds>
<intervalMinutes>60</intervalMinutes>
- <hour>12</hour>
- <minutes>0</minutes>
- <weekDay>1</weekDay>
<DayOfMonth>1</DayOfMonth>
- <parallel>N</parallel>
+ <weekDay>1</weekDay>
+ <minutes>0</minutes>
+ <hour>12</hour>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
<xloc>50</xloc>
<yloc>50</yloc>
+ <parallel>N</parallel>
<attributes_hac/>
</action>
<action>
- <name>Run Checksum Unit Test</name>
- <description/>
- <type>RunPipelineTests</type>
- <attributes/>
<test_names>
<test_name>
<name>0043-checksum UNIT</name>
</test_name>
</test_names>
- <parallel>N</parallel>
+ <name>Run Checksum Unit Test</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
<xloc>192</xloc>
<yloc>48</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <filename>${PROJECT_HOME}/0043-checksum-prefix.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <create_parent_folder>N</create_parent_folder>
+ <set_logfile>N</set_logfile>
+ <set_append_logfile>N</set_append_logfile>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <wait_until_finished>Y</wait_until_finished>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <run_configuration>local</run_configuration>
+ <name>0043-checksum-prefix</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <xloc>384</xloc>
+ <yloc>48</yloc>
+ <parallel>N</parallel>
<attributes_hac/>
</action>
</actions>
@@ -68,12 +94,18 @@ limitations under the License.
<hop>
<from>Start</from>
<to>Run Checksum Unit Test</to>
- <enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Run Checksum Unit Test</from>
+ <to>0043-checksum-prefix</to>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ <enabled>Y</enabled>
</hop>
</hops>
- <notepads>
- </notepads>
+ <notepads/>
<attributes/>
</workflow>
diff --git
a/plugins/transforms/checksum/src/main/java/org/apache/hop/pipeline/transforms/checksum/CheckSum.java
b/plugins/transforms/checksum/src/main/java/org/apache/hop/pipeline/transforms/checksum/CheckSum.java
index 1dbb8b1e23..0522333aff 100644
---
a/plugins/transforms/checksum/src/main/java/org/apache/hop/pipeline/transforms/checksum/CheckSum.java
+++
b/plugins/transforms/checksum/src/main/java/org/apache/hop/pipeline/transforms/checksum/CheckSum.java
@@ -22,6 +22,7 @@ import java.security.MessageDigest;
import java.util.zip.Adler32;
import java.util.zip.CRC32;
import org.apache.commons.codec.binary.Hex;
+import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowDataUtil;
@@ -84,6 +85,10 @@ public class CheckSum extends BaseTransform<CheckSumMeta,
CheckSumData> {
}
data.fieldnr = data.fieldnrs.length;
+ data.separator = Const.NVL(resolve(meta.getSeparator()), "");
+ data.prefix = Const.NVL(resolve(meta.getPrefix()), "");
+ data.suffix = Const.NVL(resolve(meta.getSuffix()), "");
+
try {
if (meta.getCheckSumType() == CheckSumMeta.CheckSumType.MD5
|| meta.getCheckSumType() == CheckSumMeta.CheckSumType.SHA1
@@ -155,40 +160,67 @@ public class CheckSum extends BaseTransform<CheckSumMeta,
CheckSumData> {
}
private byte[] createCheckSum(Object[] r) throws Exception {
+ byte[] byteArray = buildCheckSumInputBytes(r);
+
+ // Return null when all input values are null.
+ if (byteArray == null) {
+ return null;
+ }
+
+ // Updates the digest using the specified array of bytes
+ data.digest.update(byteArray);
+
+ // Completes the hash computation by performing final operations such as
padding
+ // After digest has been called, the MessageDigest object is reset to its
initialized state
+
+ return data.digest.digest();
+ }
+ private byte[] buildCheckSumInputBytes(Object[] r) throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ if (!appendCheckSumContent(baos, r)) {
+ return null;
+ }
+ return baos.toByteArray();
+ }
+
+ private boolean appendCheckSumContent(ByteArrayOutputStream baos, Object[]
r) throws Exception {
boolean valueAdded = false;
+ boolean fieldValueAdded = false;
+
+ if (!Utils.isEmpty(data.prefix)) {
+ baos.write(data.prefix.getBytes());
+ valueAdded = true;
+ }
// Loop through fields
for (int i = 0; i < data.fieldnr; i++) {
IValueMeta valueMeta = getInputRowMeta().getValueMeta(data.fieldnrs[i]);
+ byte[] fieldBytes = null;
if (valueMeta.isBinary()) {
- byte[] bytes = getInputRowMeta().getBinary(r, data.fieldnrs[i]);
- if (bytes != null) {
- valueAdded = true;
- baos.write(bytes);
- }
+ fieldBytes = getInputRowMeta().getBinary(r, data.fieldnrs[i]);
} else {
Object value = valueMeta.getNativeDataType(r[data.fieldnrs[i]]);
if (value != null) {
- valueAdded = true;
- baos.write(value.toString().getBytes());
+ fieldBytes = value.toString().getBytes();
+ }
+ }
+ if (fieldBytes != null) {
+ if (fieldValueAdded && !Utils.isEmpty(data.separator)) {
+ baos.write(data.separator.getBytes());
}
+ baos.write(fieldBytes);
+ fieldValueAdded = true;
+ valueAdded = true;
}
}
- // Return null when all input values are null.
- if (!valueAdded) {
- return null;
+ if (!Utils.isEmpty(data.suffix)) {
+ baos.write(data.suffix.getBytes());
+ valueAdded = true;
}
- // Updates the digest using the specified array of bytes
- data.digest.update(baos.toByteArray());
-
- // Completes the hash computation by performing final operations such as
padding
- // After digest has been called, the MessageDigest object is reset to its
initialized state
-
- return data.digest.digest();
+ return valueAdded;
}
private static String getStringFromBytes(byte[] bytes) {
@@ -205,37 +237,13 @@ public class CheckSum extends BaseTransform<CheckSumMeta,
CheckSumData> {
private Long calculCheckSum(Object[] r) throws Exception {
Long retval;
- byte[] byteArray;
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- boolean valueAdded = false;
-
- // Loop through fields
- for (int i = 0; i < data.fieldnr; i++) {
- IValueMeta valueMeta = getInputRowMeta().getValueMeta(data.fieldnrs[i]);
-
- if (valueMeta.isBinary()) {
- byte[] bytes = getInputRowMeta().getBinary(r, data.fieldnrs[i]);
- if (bytes != null) {
- valueAdded = true;
- baos.write(bytes);
- }
- } else {
- Object value = valueMeta.getNativeDataType(r[data.fieldnrs[i]]);
- if (value != null) {
- valueAdded = true;
- baos.write(value.toString().getBytes());
- }
- }
- }
+ byte[] byteArray = buildCheckSumInputBytes(r);
// Return null when all input values are null.
- if (!valueAdded) {
+ if (byteArray == null) {
return null;
}
- byteArray = baos.toByteArray();
-
if (meta.getCheckSumType() == CheckSumMeta.CheckSumType.CRC32) {
CRC32 crc32 = new CRC32();
crc32.update(byteArray);
diff --git
a/plugins/transforms/checksum/src/main/java/org/apache/hop/pipeline/transforms/checksum/CheckSumData.java
b/plugins/transforms/checksum/src/main/java/org/apache/hop/pipeline/transforms/checksum/CheckSumData.java
index da46ac01ab..9f04031f2a 100644
---
a/plugins/transforms/checksum/src/main/java/org/apache/hop/pipeline/transforms/checksum/CheckSumData.java
+++
b/plugins/transforms/checksum/src/main/java/org/apache/hop/pipeline/transforms/checksum/CheckSumData.java
@@ -35,6 +35,12 @@ public class CheckSumData extends BaseTransformData
implements ITransformData {
public MessageDigest digest;
+ public String separator;
+
+ public String prefix;
+
+ public String suffix;
+
public CheckSumData() {
super();
this.nrInfields = 0;
diff --git
a/plugins/transforms/checksum/src/main/java/org/apache/hop/pipeline/transforms/checksum/CheckSumDialog.java
b/plugins/transforms/checksum/src/main/java/org/apache/hop/pipeline/transforms/checksum/CheckSumDialog.java
index cf6fbdc910..7332319fc6 100644
---
a/plugins/transforms/checksum/src/main/java/org/apache/hop/pipeline/transforms/checksum/CheckSumDialog.java
+++
b/plugins/transforms/checksum/src/main/java/org/apache/hop/pipeline/transforms/checksum/CheckSumDialog.java
@@ -33,6 +33,7 @@ import org.apache.hop.ui.core.dialog.BaseDialog;
import org.apache.hop.ui.core.dialog.ErrorDialog;
import org.apache.hop.ui.core.widget.ColumnInfo;
import org.apache.hop.ui.core.widget.TableView;
+import org.apache.hop.ui.core.widget.TextVar;
import org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
import org.apache.hop.ui.pipeline.transform.ITableItemInsertListener;
import org.eclipse.swt.SWT;
@@ -58,6 +59,12 @@ public class CheckSumDialog extends BaseTransformDialog {
private Text wResult;
+ private TextVar wPrefix;
+
+ private TextVar wSeparator;
+
+ private TextVar wSuffix;
+
private ColumnInfo[] columnInfo;
private final List<String> inputFields = new ArrayList<>();
@@ -148,13 +155,64 @@ public class CheckSumDialog extends BaseTransformDialog {
fdResult.right = new FormAttachment(100, 0);
wResult.setLayoutData(fdResult);
+ // Prefix
+ Label wlPrefix = new Label(shell, SWT.RIGHT);
+ wlPrefix.setText(BaseMessages.getString(PKG,
"CheckSumDialog.Prefix.Label"));
+ PropsUi.setLook(wlPrefix);
+ FormData fdlPrefix = new FormData();
+ fdlPrefix.left = new FormAttachment(0, 0);
+ fdlPrefix.right = new FormAttachment(middle, -margin);
+ fdlPrefix.top = new FormAttachment(wResult, margin);
+ wlPrefix.setLayoutData(fdlPrefix);
+ wPrefix = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT |
SWT.BORDER);
+ PropsUi.setLook(wPrefix);
+ FormData fdPrefix = new FormData();
+ fdPrefix.left = new FormAttachment(middle, 0);
+ fdPrefix.top = new FormAttachment(wResult, margin);
+ fdPrefix.right = new FormAttachment(100, 0);
+ wPrefix.setLayoutData(fdPrefix);
+
+ // Separator
+ Label wlSeparator = new Label(shell, SWT.RIGHT);
+ wlSeparator.setText(BaseMessages.getString(PKG,
"CheckSumDialog.Separator.Label"));
+ PropsUi.setLook(wlSeparator);
+ FormData fdlSeparator = new FormData();
+ fdlSeparator.left = new FormAttachment(0, 0);
+ fdlSeparator.right = new FormAttachment(middle, -margin);
+ fdlSeparator.top = new FormAttachment(wPrefix, margin);
+ wlSeparator.setLayoutData(fdlSeparator);
+ wSeparator = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT |
SWT.BORDER);
+ PropsUi.setLook(wSeparator);
+ FormData fdSeparator = new FormData();
+ fdSeparator.left = new FormAttachment(middle, 0);
+ fdSeparator.top = new FormAttachment(wPrefix, margin);
+ fdSeparator.right = new FormAttachment(100, 0);
+ wSeparator.setLayoutData(fdSeparator);
+
+ // Suffix
+ Label wlSuffix = new Label(shell, SWT.RIGHT);
+ wlSuffix.setText(BaseMessages.getString(PKG,
"CheckSumDialog.Suffix.Label"));
+ PropsUi.setLook(wlSuffix);
+ FormData fdlSuffix = new FormData();
+ fdlSuffix.left = new FormAttachment(0, 0);
+ fdlSuffix.right = new FormAttachment(middle, -margin);
+ fdlSuffix.top = new FormAttachment(wSeparator, margin);
+ wlSuffix.setLayoutData(fdlSuffix);
+ wSuffix = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT |
SWT.BORDER);
+ PropsUi.setLook(wSuffix);
+ FormData fdSuffix = new FormData();
+ fdSuffix.left = new FormAttachment(middle, 0);
+ fdSuffix.top = new FormAttachment(wSeparator, margin);
+ fdSuffix.right = new FormAttachment(100, 0);
+ wSuffix.setLayoutData(fdSuffix);
+
// Table with fields
Label wlFields = new Label(shell, SWT.NONE);
wlFields.setText(BaseMessages.getString(PKG,
"CheckSumDialog.Fields.Label"));
PropsUi.setLook(wlFields);
FormData fdlFields = new FormData();
fdlFields.left = new FormAttachment(0, 0);
- fdlFields.top = new FormAttachment(wResult, margin);
+ fdlFields.top = new FormAttachment(wSuffix, margin);
wlFields.setLayoutData(fdlFields);
final int nrCols = 1;
@@ -260,6 +318,9 @@ public class CheckSumDialog extends BaseTransformDialog {
wType.setText(input.getCheckSumType().getDescription());
wResult.setText(Const.NVL(input.getResultFieldName(), ""));
wResultType.setText(input.getResultType().getDescription());
+ wPrefix.setText(Const.NVL(input.getPrefix(), ""));
+ wSeparator.setText(Const.NVL(input.getSeparator(), ""));
+ wSuffix.setText(Const.NVL(input.getSuffix(), ""));
Table table = wFields.table;
if (!input.getFields().isEmpty()) {
@@ -290,6 +351,9 @@ public class CheckSumDialog extends BaseTransformDialog {
input.setCheckSumType(CheckSumMeta.CheckSumType.getTypeFromDescription(wType.getText()));
input.setResultFieldName(wResult.getText());
input.setResultType(CheckSumMeta.ResultType.getTypeFromDescription(wResultType.getText()));
+ input.setPrefix(wPrefix.getText());
+ input.setSeparator(wSeparator.getText());
+ input.setSuffix(wSuffix.getText());
input.getFields().clear();
for (TableItem item : wFields.getNonEmptyItems()) {
diff --git
a/plugins/transforms/checksum/src/main/java/org/apache/hop/pipeline/transforms/checksum/CheckSumMeta.java
b/plugins/transforms/checksum/src/main/java/org/apache/hop/pipeline/transforms/checksum/CheckSumMeta.java
index 5b7339c28b..6b74f6e0e7 100644
---
a/plugins/transforms/checksum/src/main/java/org/apache/hop/pipeline/transforms/checksum/CheckSumMeta.java
+++
b/plugins/transforms/checksum/src/main/java/org/apache/hop/pipeline/transforms/checksum/CheckSumMeta.java
@@ -188,6 +188,24 @@ public class CheckSumMeta extends
BaseTransformMeta<CheckSum, CheckSumData> {
injectionKeyDescription = "CheckSum.Injection.RESULT_TYPE")
private ResultType resultType;
+ @HopMetadataProperty(
+ key = "separator",
+ injectionKey = "SEPARATOR",
+ injectionKeyDescription = "CheckSum.Injection.SEPARATOR")
+ private String separator;
+
+ @HopMetadataProperty(
+ key = "prefix",
+ injectionKey = "PREFIX",
+ injectionKeyDescription = "CheckSum.Injection.PREFIX")
+ private String prefix;
+
+ @HopMetadataProperty(
+ key = "suffix",
+ injectionKey = "SUFFIX",
+ injectionKeyDescription = "CheckSum.Injection.SUFFIX")
+ private String suffix;
+
public CheckSumMeta() {
fields = new ArrayList<>();
checkSumType = CheckSumType.CRC32;
@@ -200,6 +218,9 @@ public class CheckSumMeta extends
BaseTransformMeta<CheckSum, CheckSumData> {
meta.checkSumType = checkSumType;
meta.resultFieldName = resultFieldName;
meta.resultType = resultType;
+ meta.separator = separator;
+ meta.prefix = prefix;
+ meta.suffix = suffix;
for (Field field : fields) {
meta.fields.add(new Field(field));
}
@@ -397,4 +418,52 @@ public class CheckSumMeta extends
BaseTransformMeta<CheckSum, CheckSumData> {
public void setResultType(ResultType resultType) {
this.resultType = resultType;
}
+
+ /**
+ * Gets separator
+ *
+ * @return value of separator
+ */
+ public String getSeparator() {
+ return separator;
+ }
+
+ /**
+ * @param separator The separator to set
+ */
+ public void setSeparator(String separator) {
+ this.separator = separator;
+ }
+
+ /**
+ * Gets prefix
+ *
+ * @return value of prefix
+ */
+ public String getPrefix() {
+ return prefix;
+ }
+
+ /**
+ * @param prefix The prefix to set
+ */
+ public void setPrefix(String prefix) {
+ this.prefix = prefix;
+ }
+
+ /**
+ * Gets suffix
+ *
+ * @return value of suffix
+ */
+ public String getSuffix() {
+ return suffix;
+ }
+
+ /**
+ * @param suffix The suffix to set
+ */
+ public void setSuffix(String suffix) {
+ this.suffix = suffix;
+ }
}
diff --git
a/plugins/transforms/checksum/src/main/resources/org/apache/hop/pipeline/transforms/checksum/messages/messages_en_US.properties
b/plugins/transforms/checksum/src/main/resources/org/apache/hop/pipeline/transforms/checksum/messages/messages_en_US.properties
index 237303b89f..9754ea8a15 100644
---
a/plugins/transforms/checksum/src/main/resources/org/apache/hop/pipeline/transforms/checksum/messages/messages_en_US.properties
+++
b/plugins/transforms/checksum/src/main/resources/org/apache/hop/pipeline/transforms/checksum/messages/messages_en_US.properties
@@ -20,15 +20,21 @@ CheckSum.Error.Digest=Error setting digest!
CheckSum.Error.ResultFieldMissing=Result fieldname is missing!
CheckSum.ErrorInTransformRunning=Error running transform
CheckSum.Injection.FIELD_NAME=The name of the field.
+CheckSum.Injection.PREFIX=The prefix string to prepend before the field values.
CheckSum.Injection.RESULT_FIELD=The name of the result field containing the
checksum.
CheckSum.Injection.RESULT_TYPE=Checksum result type (String, Hexadecimal and
Binary).
+CheckSum.Injection.SEPARATOR=The separator string to use between field values.
+CheckSum.Injection.SUFFIX=The suffix string to append after the field values.
CheckSum.Injection.TYPE=The type of checksum that needs to be calculated.
CheckSum.Log.CanNotFindField=Can not find field [{0}] in the input stream!
CheckSum.Log.LineNumber=Linenr {0}
CheckSum.Name=Add a checksum
CheckSumDialog.Fieldname.Column=Field
CheckSumDialog.Fields.Label=Fields used in checksum
+CheckSumDialog.Prefix.Label=Prefix
CheckSumDialog.Result.Label=Result field
+CheckSumDialog.Separator.Label=Separator
+CheckSumDialog.Suffix.Label=Suffix
CheckSumDialog.ResultType.Label=Result type
CheckSumDialog.Shell.Title=Add a checksum
CheckSumDialog.TransformName.Label=Transform name
diff --git
a/plugins/transforms/checksum/src/test/java/org/apache/hop/pipeline/transforms/checksum/CheckSumMetaInjectionTest.java
b/plugins/transforms/checksum/src/test/java/org/apache/hop/pipeline/transforms/checksum/CheckSumMetaInjectionTest.java
index ba5b9847fe..fead0c954f 100644
---
a/plugins/transforms/checksum/src/test/java/org/apache/hop/pipeline/transforms/checksum/CheckSumMetaInjectionTest.java
+++
b/plugins/transforms/checksum/src/test/java/org/apache/hop/pipeline/transforms/checksum/CheckSumMetaInjectionTest.java
@@ -38,6 +38,9 @@ class CheckSumMetaInjectionTest extends
BaseMetadataInjectionTestJunit5<CheckSum
check("RESULT_FIELD", () -> meta.getResultFieldName());
check("TYPE", () -> meta.getCheckSumType(),
CheckSumMeta.CheckSumType.class);
check("RESULT_TYPE", () -> meta.getResultType(),
CheckSumMeta.ResultType.class);
+ check("SEPARATOR", () -> meta.getSeparator());
+ check("PREFIX", () -> meta.getPrefix());
+ check("SUFFIX", () -> meta.getSuffix());
check("FIELD_NAME", () -> meta.getFields().get(0).getName());
}
}
diff --git
a/plugins/transforms/checksum/src/test/java/org/apache/hop/pipeline/transforms/checksum/CheckSumMetaTest.java
b/plugins/transforms/checksum/src/test/java/org/apache/hop/pipeline/transforms/checksum/CheckSumMetaTest.java
index 1ea283311d..8669d6424e 100644
---
a/plugins/transforms/checksum/src/test/java/org/apache/hop/pipeline/transforms/checksum/CheckSumMetaTest.java
+++
b/plugins/transforms/checksum/src/test/java/org/apache/hop/pipeline/transforms/checksum/CheckSumMetaTest.java
@@ -58,7 +58,14 @@ class CheckSumMetaTest implements IInitializer<CheckSumMeta>
{
@Test
void testSerialization() throws HopException {
List<String> attributes =
- Arrays.asList("fields", "resultFieldName", "checkSumType",
"resultType");
+ Arrays.asList(
+ "fields",
+ "resultFieldName",
+ "checkSumType",
+ "resultType",
+ "separator",
+ "prefix",
+ "suffix");
Map<String, String> getterMap = new HashMap<>();
Map<String, String> setterMap = new HashMap<>();
diff --git
a/plugins/transforms/checksum/src/test/java/org/apache/hop/pipeline/transforms/checksum/CheckSumTest.java
b/plugins/transforms/checksum/src/test/java/org/apache/hop/pipeline/transforms/checksum/CheckSumTest.java
index 853f4218ca..436f3ca8c2 100644
---
a/plugins/transforms/checksum/src/test/java/org/apache/hop/pipeline/transforms/checksum/CheckSumTest.java
+++
b/plugins/transforms/checksum/src/test/java/org/apache/hop/pipeline/transforms/checksum/CheckSumTest.java
@@ -87,20 +87,22 @@ class CheckSumTest {
}
}
- private Pipeline buildHexadecimalChecksumPipeline(CheckSumMeta.CheckSumType
checkSumType) {
+ private Pipeline buildHexadecimalChecksumPipeline(
+ CheckSumMeta.CheckSumType checkSumType, CheckSumMeta meta) {
// Create a new pipeline...
PipelineMeta pipelineMeta = new PipelineMeta();
pipelineMeta.setName(getClass().getName());
// Create a CheckSum Transform
String checkSumTransformName = "CheckSum";
- CheckSumMeta meta = new CheckSumMeta();
// Set required fields
meta.setResultFieldName("hex");
meta.setCheckSumType(checkSumType);
meta.setResultType(CheckSumMeta.ResultType.HEXADECIMAL);
- meta.setFields(List.of(new Field("test")));
+ if (meta.getFields().isEmpty()) {
+ meta.setFields(List.of(new Field("test")));
+ }
String checkSumPluginPid =
PluginRegistry.getInstance().getPluginId(TransformPluginType.class,
meta);
@@ -174,7 +176,27 @@ class CheckSumTest {
*/
private MockRowListener executeHexTest(
CheckSumMeta.CheckSumType checkSumType, Object input, IValueMeta meta)
throws Exception {
- Pipeline pipeline = buildHexadecimalChecksumPipeline(checkSumType);
+ CheckSumMeta checkSumMeta = new CheckSumMeta();
+ return executeHexTest(checkSumType, input, meta, checkSumMeta);
+ }
+
+ private MockRowListener executeHexTest(
+ CheckSumMeta.CheckSumType checkSumType,
+ Object input,
+ IValueMeta meta,
+ CheckSumMeta checkSumMeta)
+ throws Exception {
+ return executeHexTest(
+ checkSumType, new Object[] {input}, createStringRowMeta(meta),
checkSumMeta);
+ }
+
+ private MockRowListener executeHexTest(
+ CheckSumMeta.CheckSumType checkSumType,
+ Object[] input,
+ RowMeta inputRowMeta,
+ CheckSumMeta checkSumMeta)
+ throws Exception {
+ Pipeline pipeline = buildHexadecimalChecksumPipeline(checkSumType,
checkSumMeta);
pipeline.prepareExecution();
@@ -183,12 +205,11 @@ class CheckSumTest {
output.addRowListener(listener);
RowProducer rp = pipeline.addRowProducer("CheckSum", 0);
- RowMeta inputRowMeta = createStringRowMeta(meta);
((BaseTransform) pipeline.getRunThread("CheckSum",
0)).setInputRowMeta(inputRowMeta);
pipeline.startThreads();
- rp.putRow(inputRowMeta, new Object[] {input});
+ rp.putRow(inputRowMeta, input);
rp.finished();
pipeline.waitUntilFinished();
@@ -197,6 +218,33 @@ class CheckSumTest {
return listener;
}
+ @Test
+ void testHexOutput_md5WithPrefixSeparatorSuffix() throws Exception {
+ CheckSumMeta checkSumMeta = new CheckSumMeta();
+ checkSumMeta.setPrefix("CUST");
+ checkSumMeta.setSeparator("|");
+ checkSumMeta.setFields(List.of(new Field("field1"), new Field("field2")));
+
+ RowMeta inputRowMeta = new RowMeta();
+ inputRowMeta.addValueMeta(new ValueMetaString("field1"));
+ inputRowMeta.addValueMeta(new ValueMetaString("field2"));
+
+ MockRowListener results =
+ executeHexTest(
+ CheckSumMeta.CheckSumType.MD5, new Object[] {"123", "456"},
inputRowMeta, checkSumMeta);
+ assertEquals(1, results.getWritten().size());
+ assertEquals("2be9b4ef70bfdf76000b88c22201d74e",
results.getWritten().get(0)[2]);
+
+ checkSumMeta = new CheckSumMeta();
+ checkSumMeta.setPrefix("PROD");
+ checkSumMeta.setFields(List.of(new Field("field1")));
+ results =
+ executeHexTest(
+ CheckSumMeta.CheckSumType.MD5, "123", new
ValueMetaString("field1"), checkSumMeta);
+ assertEquals(1, results.getWritten().size());
+ assertEquals("f096da61d5fd73a19569a76c53998200",
results.getWritten().get(0)[1]);
+ }
+
@Test
void testHexOutput_md5() throws Exception {
MockRowListener results =
