This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/master by this push:
new 691be10eaa HOP-4126 added calculater base64 encode/decode IT
new 908d7d888e Merge pull request #1640 from bamaer/HOP-4126
691be10eaa is described below
commit 691be10eaa2d08bd08ad78dcba55e3a86fac853c
Author: Bart Maertens <[email protected]>
AuthorDate: Thu Aug 18 09:07:40 2022 +0200
HOP-4126 added calculater base64 encode/decode IT
---
.../transforms/0011-calculator-base64.hpl | 334 +++++++++++++++++++++
.../datasets/golden-calculator-base64.csv | 2 +
.../metadata/dataset/golden-calculator-base64.json | 56 ++++
.../unit-test/0011-calculator-base64 UNIT.json | 53 ++++
4 files changed, 445 insertions(+)
diff --git a/integration-tests/transforms/0011-calculator-base64.hpl
b/integration-tests/transforms/0011-calculator-base64.hpl
new file mode 100644
index 0000000000..3295d3a7f8
--- /dev/null
+++ b/integration-tests/transforms/0011-calculator-base64.hpl
@@ -0,0 +1,334 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0011-calculator-base64</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2022/08/17 19:25:39.018</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2022/08/17 19:25:39.018</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>encode</from>
+ <to>decode</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>generate 1 row</from>
+ <to>encode</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>decode</from>
+ <to>clean</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>clean</from>
+ <to>verify</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>clean</name>
+ <type>SelectValues</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>decodedBool</name>
+ </field>
+ <field>
+ <name>decodedDate</name>
+ </field>
+ <field>
+ <name>decodedInt</name>
+ </field>
+ <field>
+ <name>decodedNull</name>
+ </field>
+ <field>
+ <name>decodedNumber</name>
+ </field>
+ <field>
+ <name>decodedString</name>
+ </field>
+ <select_unspecified>N</select_unspecified>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>496</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>decode</name>
+ <type>Calculator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <failIfNoFile>Y</failIfNoFile>
+ <calculation>
+ <calc_type>BASE64_DECODE</calc_type>
+ <field_a>encodedBool</field_a>
+ <field_name>decodedBool</field_name>
+ <remove>N</remove>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ <value_type>Boolean</value_type>
+ </calculation>
+ <calculation>
+ <calc_type>BASE64_DECODE</calc_type>
+ <field_a>encodedDate</field_a>
+ <field_name>decodedDate</field_name>
+ <remove>N</remove>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ <value_type>String</value_type>
+ </calculation>
+ <calculation>
+ <calc_type>BASE64_DECODE</calc_type>
+ <field_a>encodedInt</field_a>
+ <field_name>decodedInt</field_name>
+ <remove>N</remove>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ <value_type>Integer</value_type>
+ </calculation>
+ <calculation>
+ <calc_type>BASE64_DECODE</calc_type>
+ <field_a>encodedNull</field_a>
+ <field_name>decodedNull</field_name>
+ <remove>N</remove>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ <value_type>Integer</value_type>
+ </calculation>
+ <calculation>
+ <calc_type>BASE64_DECODE</calc_type>
+ <field_a>encodedNumber</field_a>
+ <field_name>decodedNumber</field_name>
+ <remove>N</remove>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ <value_type>Number</value_type>
+ </calculation>
+ <calculation>
+ <calc_type>BASE64_DECODE</calc_type>
+ <field_a>encodedString</field_a>
+ <field_name>decodedString</field_name>
+ <remove>N</remove>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ <value_type>String</value_type>
+ </calculation>
+ <attributes/>
+ <GUI>
+ <xloc>368</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>encode</name>
+ <type>Calculator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <failIfNoFile>Y</failIfNoFile>
+ <calculation>
+ <calc_type>BASE64_ENCODE</calc_type>
+ <field_a>inputBool</field_a>
+ <field_name>encodedBool</field_name>
+ <remove>N</remove>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ </calculation>
+ <calculation>
+ <calc_type>BASE64_ENCODE</calc_type>
+ <field_a>inputDate</field_a>
+ <field_name>encodedDate</field_name>
+ <remove>N</remove>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ </calculation>
+ <calculation>
+ <calc_type>BASE64_ENCODE</calc_type>
+ <field_a>inputInt</field_a>
+ <field_name>encodedInt</field_name>
+ <remove>N</remove>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ <value_type>String</value_type>
+ </calculation>
+ <calculation>
+ <calc_type>BASE64_ENCODE</calc_type>
+ <field_a>inputNull</field_a>
+ <field_name>encodedNull</field_name>
+ <remove>N</remove>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ <value_type>String</value_type>
+ </calculation>
+ <calculation>
+ <calc_type>BASE64_ENCODE</calc_type>
+ <field_a>inputNumber</field_a>
+ <field_name>encodedNumber</field_name>
+ <remove>N</remove>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ </calculation>
+ <calculation>
+ <calc_type>BASE64_ENCODE</calc_type>
+ <field_a>inputString</field_a>
+ <field_name>encodedString</field_name>
+ <remove>N</remove>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ </calculation>
+ <attributes/>
+ <GUI>
+ <xloc>240</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>generate 1 row</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <length>-1</length>
+ <name>inputBool</name>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <type>Boolean</type>
+ <nullif>false</nullif>
+ </field>
+ <field>
+ <format>yyyy-MM-dd</format>
+ <length>-1</length>
+ <name>inputDate</name>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <type>Date</type>
+ <nullif>2022-08-17</nullif>
+ </field>
+ <field>
+ <length>-1</length>
+ <name>inputInt</name>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <type>Integer</type>
+ <nullif>123456</nullif>
+ </field>
+ <field>
+ <length>-1</length>
+ <name>inputNumber</name>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <type>Number</type>
+ <nullif>12.34</nullif>
+ </field>
+ <field>
+ <length>-1</length>
+ <name>inputNull</name>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <name>inputString</name>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <type>String</type>
+ <nullif>abcd</nullif>
+ </field>
+ </fields>
+ <interval_in_ms>5000</interval_in_ms>
+ <last_time_field>FiveSecondsAgo</last_time_field>
+ <never_ending>N</never_ending>
+ <limit>1</limit>
+ <row_time_field>now</row_time_field>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>verify</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>608</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/datasets/golden-calculator-base64.csv
b/integration-tests/transforms/datasets/golden-calculator-base64.csv
new file mode 100644
index 0000000000..687e2066d6
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-calculator-base64.csv
@@ -0,0 +1,2 @@
+decodedBool,decodedDate,decodedInt,decodedNull,decodedNumber,decodedString
+N,Wed Aug 17 00:00:00 CEST 2022,123456,,12.34,abcd
diff --git
a/integration-tests/transforms/metadata/dataset/golden-calculator-base64.json
b/integration-tests/transforms/metadata/dataset/golden-calculator-base64.json
new file mode 100644
index 0000000000..cc1fbf3193
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-calculator-base64.json
@@ -0,0 +1,56 @@
+{
+ "base_filename": "golden-calculator-base64.csv",
+ "name": "golden-calculator-base64",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "Base64 Decode",
+ "field_length": -1,
+ "field_type": 4,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "decodedBool"
+ },
+ {
+ "field_comment": "Base64 Decode",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "decodedDate"
+ },
+ {
+ "field_comment": "Base64 Decode",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_format": "####0;-####0",
+ "field_name": "decodedInt"
+ },
+ {
+ "field_comment": "Base64 Decode",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_format": "####0;-####0",
+ "field_name": "decodedNull"
+ },
+ {
+ "field_comment": "Base64 Decode",
+ "field_length": -1,
+ "field_type": 1,
+ "field_precision": -1,
+ "field_format": "####0.0#########;-####0.0#########",
+ "field_name": "decodedNumber"
+ },
+ {
+ "field_comment": "Base64 Decode",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "decodedString"
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/unit-test/0011-calculator-base64
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0011-calculator-base64
UNIT.json
new file mode 100644
index 0000000000..89482a7241
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0011-calculator-base64
UNIT.json
@@ -0,0 +1,53 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": true,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "decodedBool",
+ "data_set_field": "decodedBool"
+ },
+ {
+ "transform_field": "decodedDate",
+ "data_set_field": "decodedDate"
+ },
+ {
+ "transform_field": "decodedInt",
+ "data_set_field": "decodedInt"
+ },
+ {
+ "transform_field": "decodedNull",
+ "data_set_field": "decodedNull"
+ },
+ {
+ "transform_field": "decodedNumber",
+ "data_set_field": "decodedNumber"
+ },
+ {
+ "transform_field": "decodedString",
+ "data_set_field": "decodedString"
+ }
+ ],
+ "field_order": [
+ "decodedBool",
+ "decodedDate",
+ "decodedInt",
+ "decodedNull",
+ "decodedNumber",
+ "decodedString"
+ ],
+ "transform_name": "verify",
+ "data_set_name": "golden-calculator-base64"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0011-calculator-base64 UNIT",
+ "description": "",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename": "./0011-calculator-base64.hpl",
+ "test_type": "UNIT_TEST"
+}
\ No newline at end of file
