This is an automated email from the ASF dual-hosted git repository.
mcasters pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hop.git
The following commit(s) were added to refs/heads/master by this push:
new 2e42289 HOP-2918 : Create transform integration tests: Replace in
string
new 9b509d9 Merge pull request #841 from mattcasters/master
2e42289 is described below
commit 2e42289477569e57e063a692a1dae4a2fbd4e2e7
Author: Matt Casters <[email protected]>
AuthorDate: Tue Jun 1 21:39:22 2021 +0200
HOP-2918 : Create transform integration tests: Replace in string
---
.../transforms/0025-null-if-basic.hpl | 36 ++--
.../transforms/0026-replace-in-string.hpl | 236 +++++++++++++++++++++
.../datasets/golden-replace-in-string.csv | 5 +
.../transforms/main-0026-replace-in-string.hwf | 61 ++++++
.../metadata/dataset/golden-replace-in-string.json | 40 ++++
.../unit-test/0024-if-null-all-fields UNIT.json | 2 +-
.../unit-test/0024-if-null-by-type UNIT.json | 2 +-
.../0024-if-null-specific-fields UNIT.json | 2 +-
.../unit-test/0025-null-if-basic UNIT.json | 2 +-
... UNIT.json => 0026-replace-in-string UNIT.json} | 28 +--
10 files changed, 378 insertions(+), 36 deletions(-)
diff --git a/integration-tests/transforms/0025-null-if-basic.hpl
b/integration-tests/transforms/0025-null-if-basic.hpl
index 5e88880..38ee48c 100644
--- a/integration-tests/transforms/0025-null-if-basic.hpl
+++ b/integration-tests/transforms/0025-null-if-basic.hpl
@@ -16,7 +16,7 @@
<created_date>2021/06/01 17:40:11.867</created_date>
<modified_user>-</modified_user>
<modified_date>2021/06/01 17:40:11.867</modified_date>
- <key_for_session_key/>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
<is_key_private>N</is_key_private>
</info>
<notepads>
@@ -69,6 +69,23 @@
</GUI>
</transform>
<transform>
+ <name>Preview</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>416</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
<name>Sample data</name>
<type>DataGrid</type>
<description/>
@@ -157,23 +174,6 @@
<yloc>128</yloc>
</GUI>
</transform>
- <transform>
- <name>Preview</name>
- <type>Dummy</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
- <attributes/>
- <GUI>
- <xloc>416</xloc>
- <yloc>128</yloc>
- </GUI>
- </transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>
diff --git a/integration-tests/transforms/0026-replace-in-string.hpl
b/integration-tests/transforms/0026-replace-in-string.hpl
new file mode 100644
index 0000000..41e4193
--- /dev/null
+++ b/integration-tests/transforms/0026-replace-in-string.hpl
@@ -0,0 +1,236 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<pipeline>
+ <info>
+ <name>0026-replace-in-string</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2021/06/01 17:40:11.867</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/06/01 17:40:11.867</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Sample data</from>
+ <to>Replace in string</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Replace in string</from>
+ <to>Preview</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Preview</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>432</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Replace in string</name>
+ <type>ReplaceString</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <in_stream_name>strA</in_stream_name>
+ <out_stream_name/>
+ <use_regex>no</use_regex>
+ <replace_string>HOP</replace_string>
+ <replace_by_string> Apache Hop </replace_by_string>
+ <set_empty_string>N</set_empty_string>
+ <replace_field_by_string/>
+ <whole_word>no</whole_word>
+ <case_sensitive>no</case_sensitive>
+ <is_unicode>no</is_unicode>
+ </field>
+ <field>
+ <in_stream_name>strB</in_stream_name>
+ <out_stream_name/>
+ <use_regex>yes</use_regex>
+ <replace_string>\s*</replace_string>
+ <replace_by_string> </replace_by_string>
+ <set_empty_string>N</set_empty_string>
+ <replace_field_by_string/>
+ <whole_word>no</whole_word>
+ <case_sensitive>no</case_sensitive>
+ <is_unicode>no</is_unicode>
+ </field>
+ <field>
+ <in_stream_name>strB</in_stream_name>
+ <out_stream_name/>
+ <use_regex>yes</use_regex>
+ <replace_string>^\s</replace_string>
+ <replace_by_string/>
+ <set_empty_string>N</set_empty_string>
+ <replace_field_by_string/>
+ <whole_word>no</whole_word>
+ <case_sensitive>no</case_sensitive>
+ <is_unicode>no</is_unicode>
+ </field>
+ <field>
+ <in_stream_name>strA</in_stream_name>
+ <out_stream_name/>
+ <use_regex>yes</use_regex>
+ <replace_string>\s</replace_string>
+ <replace_by_string/>
+ <set_empty_string>N</set_empty_string>
+ <replace_field_by_string/>
+ <whole_word>no</whole_word>
+ <case_sensitive>no</case_sensitive>
+ <is_unicode>no</is_unicode>
+ </field>
+ <field>
+ <in_stream_name>strC</in_stream_name>
+ <out_stream_name/>
+ <use_regex>yes</use_regex>
+ <replace_string>[cioksx]</replace_string>
+ <replace_by_string/>
+ <set_empty_string>N</set_empty_string>
+ <replace_field_by_string/>
+ <whole_word>no</whole_word>
+ <case_sensitive>no</case_sensitive>
+ <is_unicode>no</is_unicode>
+ </field>
+ <field>
+ <in_stream_name>strD</in_stream_name>
+ <out_stream_name/>
+ <use_regex>yes</use_regex>
+ <replace_string>[CIOKSX]</replace_string>
+ <replace_by_string/>
+ <set_empty_string>N</set_empty_string>
+ <replace_field_by_string/>
+ <whole_word>no</whole_word>
+ <case_sensitive>no</case_sensitive>
+ <is_unicode>no</is_unicode>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>272</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Sample data</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>strA</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>strB</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>strC</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>strD</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>sadfsadfHOPsdfjasdf</item>
+ <item>a b c d e f</item>
+ <item>abcdef</item>
+ <item>ABCDEF</item>
+ </line>
+ <line>
+ <item>HOPsdsadfasdf</item>
+ <item>g h i j k l</item>
+ <item>ghijkl</item>
+ <item>GHIJKL</item>
+ </line>
+ <line>
+ <item>asdlfsdfHOP</item>
+ <item>m n o p q r </item>
+ <item>mnopqr</item>
+ <item>MNOPQR</item>
+ </line>
+ <line>
+ <item>HOP</item>
+ <item> s t u v w x </item>
+ <item>stuvwx</item>
+ <item>STUVWX</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/datasets/golden-replace-in-string.csv
b/integration-tests/transforms/datasets/golden-replace-in-string.csv
new file mode 100644
index 0000000..be87be4
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-replace-in-string.csv
@@ -0,0 +1,5 @@
+strA,strB,strC,strD
+sadfsadfApacheHopsdfjasdf,"a b c d e f ",abdef,ABDEF
+ApacheHopsdsadfasdf,"g h i j k l ",ghjl,GHJL
+asdlfsdfApacheHop,"m n o p q r ",mnpqr,MNPQR
+ApacheHop," s t u v w x ",tuvw,TUVW
diff --git a/integration-tests/transforms/main-0026-replace-in-string.hwf
b/integration-tests/transforms/main-0026-replace-in-string.hwf
new file mode 100644
index 0000000..c1047d3
--- /dev/null
+++ b/integration-tests/transforms/main-0026-replace-in-string.hwf
@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<workflow>
+ <name>main-0026-replace-in-string</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2021/06/01 17:45:49.992</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/06/01 17:45:49.992</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <xloc>128</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run Replace in string Tests</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0026-replace-in-string UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>336</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Run Replace in string Tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-replace-in-string.json
b/integration-tests/transforms/metadata/dataset/golden-replace-in-string.json
new file mode 100644
index 0000000..5876bdc
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-replace-in-string.json
@@ -0,0 +1,40 @@
+{
+ "base_filename": "golden-replace-in-string.csv",
+ "name": "golden-replace-in-string",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "strA"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "strB"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "strC"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "strD"
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/unit-test/0024-if-null-all-fields
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0024-if-null-all-fields
UNIT.json
index f8b1d6e..0952536 100644
--- a/integration-tests/transforms/metadata/unit-test/0024-if-null-all-fields
UNIT.json
+++ b/integration-tests/transforms/metadata/unit-test/0024-if-null-all-fields
UNIT.json
@@ -48,6 +48,6 @@
"description": "",
"trans_test_tweaks": [],
"persist_filename": "",
- "pipeline_filename":
"/home/matt/git/mattcasters/incubator-hop/integration-tests/transforms/0024-if-null-all-fields.hpl",
+ "pipeline_filename": "./0024-if-null-all-fields.hpl",
"test_type": "UNIT_TEST"
}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/unit-test/0024-if-null-by-type
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0024-if-null-by-type UNIT.json
index 722a2fc..15e5054 100644
--- a/integration-tests/transforms/metadata/unit-test/0024-if-null-by-type
UNIT.json
+++ b/integration-tests/transforms/metadata/unit-test/0024-if-null-by-type
UNIT.json
@@ -48,6 +48,6 @@
"description": "",
"trans_test_tweaks": [],
"persist_filename": "",
- "pipeline_filename":
"/home/matt/git/mattcasters/incubator-hop/integration-tests/transforms/0024-if-null-by-type.hpl",
+ "pipeline_filename": "./0024-if-null-by-type.hpl",
"test_type": "UNIT_TEST"
}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/unit-test/0024-if-null-specific-fields
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0024-if-null-specific-fields
UNIT.json
index e0cc4f7..def5663 100644
---
a/integration-tests/transforms/metadata/unit-test/0024-if-null-specific-fields
UNIT.json
+++
b/integration-tests/transforms/metadata/unit-test/0024-if-null-specific-fields
UNIT.json
@@ -48,6 +48,6 @@
"description": "",
"trans_test_tweaks": [],
"persist_filename": "",
- "pipeline_filename":
"/home/matt/git/mattcasters/incubator-hop/integration-tests/transforms/0024-if-null-specific-fields.hpl",
+ "pipeline_filename": "./0024-if-null-specific-fields.hpl",
"test_type": "UNIT_TEST"
}
\ No newline at end of file
diff --git a/integration-tests/transforms/metadata/unit-test/0025-null-if-basic
UNIT.json b/integration-tests/transforms/metadata/unit-test/0025-null-if-basic
UNIT.json
index 1179c66..7dfd8a7 100644
--- a/integration-tests/transforms/metadata/unit-test/0025-null-if-basic
UNIT.json
+++ b/integration-tests/transforms/metadata/unit-test/0025-null-if-basic
UNIT.json
@@ -38,6 +38,6 @@
"description": "",
"trans_test_tweaks": [],
"persist_filename": "",
- "pipeline_filename":
"/home/matt/git/mattcasters/incubator-hop/integration-tests/transforms/0025-null-if-basic.hpl",
+ "pipeline_filename": "./0025-null-if-basic.hpl",
"test_type": "UNIT_TEST"
}
\ No newline at end of file
diff --git a/integration-tests/transforms/metadata/unit-test/0025-null-if-basic
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0026-replace-in-string
UNIT.json
similarity index 53%
copy from integration-tests/transforms/metadata/unit-test/0025-null-if-basic
UNIT.json
copy to integration-tests/transforms/metadata/unit-test/0026-replace-in-string
UNIT.json
index 1179c66..df112c0 100644
--- a/integration-tests/transforms/metadata/unit-test/0025-null-if-basic
UNIT.json
+++ b/integration-tests/transforms/metadata/unit-test/0026-replace-in-string
UNIT.json
@@ -7,37 +7,37 @@
{
"field_mappings": [
{
- "transform_field": "boolD",
- "data_set_field": "boolD"
+ "transform_field": "strA",
+ "data_set_field": "strA"
},
{
- "transform_field": "intB",
- "data_set_field": "intB"
+ "transform_field": "strB",
+ "data_set_field": "strB"
},
{
- "transform_field": "numC",
- "data_set_field": "numC"
+ "transform_field": "strC",
+ "data_set_field": "strC"
},
{
- "transform_field": "strA",
- "data_set_field": "strA"
+ "transform_field": "strD",
+ "data_set_field": "strD"
}
],
"field_order": [
"strA",
- "intB",
- "numC",
- "boolD"
+ "strB",
+ "strC",
+ "strD"
],
"transform_name": "Preview",
- "data_set_name": "golden-null-if-basic"
+ "data_set_name": "golden-replace-in-string"
}
],
"input_data_sets": [],
- "name": "0025-null-if-basic UNIT",
+ "name": "0026-replace-in-string UNIT",
"description": "",
"trans_test_tweaks": [],
"persist_filename": "",
- "pipeline_filename":
"/home/matt/git/mattcasters/incubator-hop/integration-tests/transforms/0025-null-if-basic.hpl",
+ "pipeline_filename": "./0026-replace-in-string.hpl",
"test_type": "UNIT_TEST"
}
\ No newline at end of file
