This is an automated email from the ASF dual-hosted git repository.
mcasters pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hop.git
The following commit(s) were added to refs/heads/master by this push:
new 3753700 HOP-2918 : Create transform integration tests: Number Range,
If null, Null if
new 7fc7858 Merge pull request #840 from mattcasters/master
3753700 is described below
commit 3753700d9571a69f3652e75f6449d14936d47179
Author: Matt Casters <[email protected]>
AuthorDate: Tue Jun 1 17:55:51 2021 +0200
HOP-2918 : Create transform integration tests: Number Range, If null, Null
if
---
.../neo4j-graph-output-build.hpl | 284 +++++++++++++++++++++
.../neo4j-graph-output-validation.hpl | 149 +++++++++++
integration-tests/transforms/0023-number-range.hpl | 110 ++++----
.../transforms/0024-if-null-all-fields.hpl | 217 ++++++++++++++++
.../transforms/0024-if-null-by-type.hpl | 235 +++++++++++++++++
.../transforms/0024-if-null-specific-fields.hpl | 241 +++++++++++++++++
...023-number-range.hpl => 0025-null-if-basic.hpl} | 186 +++++++-------
.../datasets/golden-if-null-all-fields.csv | 7 +
.../transforms/datasets/golden-if-null-by-type.csv | 7 +
.../datasets/golden-if-null-specific-fields.csv | 7 +
.../transforms/datasets/golden-null-if-basic.csv | 5 +
.../transforms/datasets/golden-number-rang.csv | 20 +-
.../transforms/main-0023-number-range.hwf | 61 +++++
integration-tests/transforms/main-0024-if-null.hwf | 67 +++++
integration-tests/transforms/main-0025-null-if.hwf | 61 +++++
.../dataset/golden-if-null-all-fields.json | 56 ++++
.../metadata/dataset/golden-if-null-by-type.json | 56 ++++
.../dataset/golden-if-null-specific-fields.json | 56 ++++
...number-range.json => golden-null-if-basic.json} | 24 +-
.../metadata/dataset/golden-number-range.json | 2 +-
.../metadata/unit-test/0023-number-range UNIT.json | 10 +-
.../unit-test/0024-if-null-all-fields UNIT.json | 53 ++++
.../unit-test/0024-if-null-by-type UNIT.json | 53 ++++
.../0024-if-null-specific-fields UNIT.json | 53 ++++
.../unit-test/0025-null-if-basic UNIT.json | 43 ++++
25 files changed, 1888 insertions(+), 175 deletions(-)
diff --git
a/integration-tests/neo4j/tests/neo4j-graph-output/neo4j-graph-output-build.hpl
b/integration-tests/neo4j/tests/neo4j-graph-output/neo4j-graph-output-build.hpl
new file mode 100644
index 0000000..fcaf655
--- /dev/null
+++
b/integration-tests/neo4j/tests/neo4j-graph-output/neo4j-graph-output-build.hpl
@@ -0,0 +1,284 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<pipeline>
+ <info>
+ <name>neo4j-graph-output-build</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2019/08/06 16:10:49.753</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2019/08/06 16:10:49.753</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ <notepad>
+ <note>Commit size set to 123 to check for dangling records</note>
+ <xloc>160</xloc>
+ <yloc>240</yloc>
+ <width>330</width>
+ <heigth>29</heigth>
+ <fontname>Sans</fontname>
+ <fontsize>14</fontsize>
+ <fontbold>N</fontbold>
+ <fontitalic>N</fontitalic>
+ <fontcolorred>0</fontcolorred>
+ <fontcolorgreen>0</fontcolorgreen>
+ <fontcolorblue>0</fontcolorblue>
+ <backgroundcolorred>136</backgroundcolorred>
+ <backgroundcolorgreen>138</backgroundcolorgreen>
+ <backgroundcolorblue>133</backgroundcolorblue>
+ <bordercolorred>100</bordercolorred>
+ <bordercolorgreen>100</bordercolorgreen>
+ <bordercolorblue>100</bordercolorblue>
+ </notepad>
+ </notepads>
+ <order>
+ <hop>
+ <from>customers-1k.txt</from>
+ <to>Update Customers graph</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Update Customers graph</name>
+ <type>Neo4jGraphOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <connection>Test</connection>
+ <model>Customers</model>
+ <batch_size>123</batch_size>
+ <create_indexes>Y</create_indexes>
+ <returning_graph>N</returning_graph>
+ <return_graph_field/>
+ <validate_against_model>N</validate_against_model>
+ <out_of_order_allowed>Y</out_of_order_allowed>
+ <mappings>
+ <mapping>
+ <source_field>id</source_field>
+ <target_type>Node</target_type>
+ <target_name>Customer</target_name>
+ <target_property>id</target_property>
+ </mapping>
+ <mapping>
+ <source_field>name</source_field>
+ <target_type>Node</target_type>
+ <target_name>Customer</target_name>
+ <target_property>name</target_property>
+ </mapping>
+ <mapping>
+ <source_field>firstname</source_field>
+ <target_type>Node</target_type>
+ <target_name>Customer</target_name>
+ <target_property>firstname</target_property>
+ </mapping>
+ <mapping>
+ <source_field>birthdate</source_field>
+ <target_type>Node</target_type>
+ <target_name>Customer</target_name>
+ <target_property>birthdate</target_property>
+ </mapping>
+ <mapping>
+ <source_field>zip</source_field>
+ <target_type>Node</target_type>
+ <target_name>Zip</target_name>
+ <target_property>zip</target_property>
+ </mapping>
+ <mapping>
+ <source_field>city</source_field>
+ <target_type>Node</target_type>
+ <target_name>City</target_name>
+ <target_property>city</target_property>
+ </mapping>
+ <mapping>
+ <source_field>street</source_field>
+ <target_type>Node</target_type>
+ <target_name>Street</target_name>
+ <target_property>street</target_property>
+ </mapping>
+ <mapping>
+ <source_field>housenr</source_field>
+ <target_type>Node</target_type>
+ <target_name>House</target_name>
+ <target_property>housenr</target_property>
+ </mapping>
+ <mapping>
+ <source_field>stateCode</source_field>
+ <target_type>Node</target_type>
+ <target_name>State</target_name>
+ <target_property>stateCode</target_property>
+ </mapping>
+ <mapping>
+ <source_field>state</source_field>
+ <target_type>Node</target_type>
+ <target_name>State</target_name>
+ <target_property>state</target_property>
+ </mapping>
+ </mappings>
+ <attributes/>
+ <GUI>
+ <xloc>464</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>customers-1k.txt</name>
+ <type>CSVInput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <filename>${PROJECT_HOME}/datasets/customers.csv</filename>
+ <filename_field/>
+ <rownum_field/>
+ <include_filename>N</include_filename>
+ <separator>,</separator>
+ <enclosure>"</enclosure>
+ <header>Y</header>
+ <buffer_size>50000</buffer_size>
+ <lazy_conversion>N</lazy_conversion>
+ <add_filename_result>N</add_filename_result>
+ <parallel>N</parallel>
+ <newline_possible>N</newline_possible>
+ <encoding/>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <format>#</format>
+ <currency>EUR</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>15</length>
+ <precision>0</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>name</name>
+ <type>String</type>
+ <format/>
+ <currency>EUR</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>10</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>firstname</name>
+ <type>String</type>
+ <format/>
+ <currency>EUR</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>13</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>zip</name>
+ <type>Integer</type>
+ <format>#</format>
+ <currency>EUR</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>15</length>
+ <precision>0</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>city</name>
+ <type>String</type>
+ <format/>
+ <currency>EUR</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>8</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>birthdate</name>
+ <type>Date</type>
+ <format>yyyy/MM/dd</format>
+ <currency>EUR</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>street</name>
+ <type>String</type>
+ <format/>
+ <currency>EUR</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>11</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>housenr</name>
+ <type>Integer</type>
+ <format>#</format>
+ <currency>EUR</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>15</length>
+ <precision>0</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>stateCode</name>
+ <type>String</type>
+ <format/>
+ <currency>EUR</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>9</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>state</name>
+ <type>String</type>
+ <format/>
+ <currency>EUR</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>30</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>208</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/neo4j/tests/neo4j-graph-output/neo4j-graph-output-validation.hpl
b/integration-tests/neo4j/tests/neo4j-graph-output/neo4j-graph-output-validation.hpl
new file mode 100644
index 0000000..f348f03
--- /dev/null
+++
b/integration-tests/neo4j/tests/neo4j-graph-output/neo4j-graph-output-validation.hpl
@@ -0,0 +1,149 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<pipeline>
+ <info>
+ <name>neo4j-cypher-graph-validation</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2019/08/06 18:28:33.119</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2019/08/06 18:28:33.119</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Read :Customer</from>
+ <to>CUSTOMERS</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>CUSTOMERS</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>368</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Read :Customer</name>
+ <type>Neo4jCypherOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <connection>Test</connection>
+ <cypher>MATCH (c:Customer)
+MATCH (c)-[:IN_CITY]->(i:City)
+MATCH (c)-[:IN_STATE]->(s:State)
+MATCH (c)-[:IN_STREET]->(r:Street)
+MATCH (c)-[:IN_HOUSE]->(h:House)
+MATCH (c)-[:IN_ZIP]->(z:Zip)
+RETURN c.id as id,
+ c.name as name,
+ c.firstname as firstname,
+ c.birthdate as birthdate,
+ z.zip as zip,
+ i.city as city,
+ r.street as street,
+ h.housenr as housenr,
+ s.stateCode as stateCode,
+ s.state as state</cypher>
+ <batch_size/>
+ <read_only>Y</read_only>
+ <nr_retries_on_error/>
+ <retry>Y</retry>
+ <cypher_from_field>N</cypher_from_field>
+ <cypher_field/>
+ <unwind>N</unwind>
+ <unwind_map/>
+ <returning_graph>N</returning_graph>
+ <return_graph_field/>
+ <mappings/>
+ <returns>
+ <return>
+ <name>id</name>
+ <type>Integer</type>
+ <source_type>Integer</source_type>
+ </return>
+ <return>
+ <name>name</name>
+ <type>String</type>
+ <source_type>String</source_type>
+ </return>
+ <return>
+ <name>firstname</name>
+ <type>String</type>
+ <source_type>String</source_type>
+ </return>
+ <return>
+ <name>birthdate</name>
+ <type>Date</type>
+ <source_type>LocalDateTime</source_type>
+ </return>
+ <return>
+ <name>zip</name>
+ <type>Integer</type>
+ <source_type>Integer</source_type>
+ </return>
+ <return>
+ <name>city</name>
+ <type>String</type>
+ <source_type>String</source_type>
+ </return>
+ <return>
+ <name>street</name>
+ <type>String</type>
+ <source_type>String</source_type>
+ </return>
+ <return>
+ <name>housenr</name>
+ <type>Integer</type>
+ <source_type>Integer</source_type>
+ </return>
+ <return>
+ <name>stateCode</name>
+ <type>String</type>
+ <source_type>String</source_type>
+ </return>
+ <return>
+ <name>state</name>
+ <type>String</type>
+ <source_type>String</source_type>
+ </return>
+ </returns>
+ <attributes/>
+ <GUI>
+ <xloc>192</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/0023-number-range.hpl
b/integration-tests/transforms/0023-number-range.hpl
index afa9515..a2f7eda 100644
--- a/integration-tests/transforms/0023-number-range.hpl
+++ b/integration-tests/transforms/0023-number-range.hpl
@@ -16,7 +16,7 @@
<created_date>2021/05/31 14:12:11.112</created_date>
<modified_user>-</modified_user>
<modified_date>2021/05/31 14:12:11.112</modified_date>
- <key_for_session_key/>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
<is_key_private>N</is_key_private>
</info>
<notepads>
@@ -34,6 +34,53 @@
</hop>
</order>
<transform>
+ <name>Price range</name>
+ <type>NumberRange</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <inputField>price</inputField>
+ <outputField>range</outputField>
+ <fallBackValue>illegal</fallBackValue>
+ <rules>
+ <rule>
+ <lower_bound>-1.7976931348623157E308</lower_bound>
+ <upper_bound>10.0</upper_bound>
+ <value>bargain</value>
+ </rule>
+ <rule>
+ <lower_bound>10.0</lower_bound>
+ <upper_bound>50.0</upper_bound>
+ <value>cheap</value>
+ </rule>
+ <rule>
+ <lower_bound>50.0</lower_bound>
+ <upper_bound>250.0</upper_bound>
+ <value>average</value>
+ </rule>
+ <rule>
+ <lower_bound>250.0</lower_bound>
+ <upper_bound>1000.0</upper_bound>
+ <value>pricey</value>
+ </rule>
+ <rule>
+ <lower_bound>1000.0</lower_bound>
+ <upper_bound>1.7976931348623157E308</upper_bound>
+ <value>expensive</value>
+ </rule>
+ </rules>
+ <attributes/>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
<name>Prices</name>
<type>DataGrid</type>
<description/>
@@ -59,12 +106,12 @@
<field>
<name>price</name>
<type>Number</type>
- <format>#.#</format>
+ <format>#.00</format>
<currency/>
<decimal>.</decimal>
<group/>
- <length>-1</length>
- <precision>-1</precision>
+ <length>5</length>
+ <precision>2</precision>
<set_empty_string>N</set_empty_string>
</field>
</fields>
@@ -112,55 +159,8 @@
</data>
<attributes/>
<GUI>
- <xloc>160</xloc>
- <yloc>112</yloc>
- </GUI>
- </transform>
- <transform>
- <name>Price range</name>
- <type>NumberRange</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
- <inputField>price</inputField>
- <outputField>range</outputField>
- <fallBackValue>illegal</fallBackValue>
- <rules>
- <rule>
- <lower_bound>-1.7976931348623157E308</lower_bound>
- <upper_bound>10.0</upper_bound>
- <value>bargain</value>
- </rule>
- <rule>
- <lower_bound>10.0</lower_bound>
- <upper_bound>50.0</upper_bound>
- <value>cheap</value>
- </rule>
- <rule>
- <lower_bound>50.0</lower_bound>
- <upper_bound>250.0</upper_bound>
- <value>average</value>
- </rule>
- <rule>
- <lower_bound>250.0</lower_bound>
- <upper_bound>1000.0</upper_bound>
- <value>pricey</value>
- </rule>
- <rule>
- <lower_bound>1000.0</lower_bound>
- <upper_bound>1.7976931348623157E308</upper_bound>
- <value>expensive</value>
- </rule>
- </rules>
- <attributes/>
- <GUI>
- <xloc>320</xloc>
- <yloc>112</yloc>
+ <xloc>96</xloc>
+ <yloc>80</yloc>
</GUI>
</transform>
<transform>
@@ -176,8 +176,8 @@
</partitioning>
<attributes/>
<GUI>
- <xloc>480</xloc>
- <yloc>112</yloc>
+ <xloc>416</xloc>
+ <yloc>80</yloc>
</GUI>
</transform>
<transform_error_handling>
diff --git a/integration-tests/transforms/0024-if-null-all-fields.hpl
b/integration-tests/transforms/0024-if-null-all-fields.hpl
new file mode 100644
index 0000000..3cd7711
--- /dev/null
+++ b/integration-tests/transforms/0024-if-null-all-fields.hpl
@@ -0,0 +1,217 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<pipeline>
+ <info>
+ <name>0024-if-null-all-fields</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2021/06/01 15:47:03.315</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/06/01 15:47:03.315</modified_date>
+ <key_for_session_key/>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Sample data</from>
+ <to>If Null - all fields</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>If Null - all fields</from>
+ <to>Verify</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>If Null - all fields</name>
+ <type>IfNull</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <replaceAllByValue>?</replaceAllByValue>
+ <replaceAllMask/>
+ <selectFields>N</selectFields>
+ <selectValuesType>N</selectValuesType>
+ <setEmptyStringAll>N</setEmptyStringAll>
+ <valuetypes>
+ </valuetypes>
+ <fields>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>272</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Sample data</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>a</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>b</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>c</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>d</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>e</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>f</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>a1</item>
+ <item/>
+ <item>c1</item>
+ <item/>
+ <item>e1</item>
+ <item>f1</item>
+ </line>
+ <line>
+ <item/>
+ <item>b2</item>
+ <item/>
+ <item/>
+ <item/>
+ <item/>
+ </line>
+ <line>
+ <item/>
+ <item/>
+ <item>c3</item>
+ <item/>
+ <item/>
+ <item>f3</item>
+ </line>
+ <line>
+ <item/>
+ <item>b4</item>
+ <item>c4</item>
+ <item>d4</item>
+ <item/>
+ <item/>
+ </line>
+ <line>
+ <item/>
+ <item/>
+ <item/>
+ <item/>
+ <item>e5</item>
+ <item/>
+ </line>
+ <line>
+ <item>a6</item>
+ <item/>
+ <item/>
+ <item/>
+ <item/>
+ <item>f6</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Verify</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>448</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/0024-if-null-by-type.hpl
b/integration-tests/transforms/0024-if-null-by-type.hpl
new file mode 100644
index 0000000..623cf8b
--- /dev/null
+++ b/integration-tests/transforms/0024-if-null-by-type.hpl
@@ -0,0 +1,235 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<pipeline>
+ <info>
+ <name>0024-if-null-by-type</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2021/06/01 15:54:59.351</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/06/01 15:54:59.351</modified_date>
+ <key_for_session_key/>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>If Null - types</from>
+ <to>Verify</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Sample data</from>
+ <to>If Null - types</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>If Null - types</name>
+ <type>IfNull</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <replaceAllByValue>?</replaceAllByValue>
+ <replaceAllMask/>
+ <selectFields>N</selectFields>
+ <selectValuesType>Y</selectValuesType>
+ <setEmptyStringAll>N</setEmptyStringAll>
+ <valuetypes>
+ <valuetype>
+ <name>Integer</name>
+ <value>999</value>
+ <mask>#</mask>
+ <set_type_empty_string>N</set_type_empty_string>
+ </valuetype>
+ <valuetype>
+ <name>Number</name>
+ <value>0.5</value>
+ <mask>#.#</mask>
+ <set_type_empty_string>N</set_type_empty_string>
+ </valuetype>
+ <valuetype>
+ <name>String</name>
+ <value>?</value>
+ <mask/>
+ <set_type_empty_string>N</set_type_empty_string>
+ </valuetype>
+ </valuetypes>
+ <fields>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>272</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Sample data</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>strA</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>intB</name>
+ <type>Integer</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>strC</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>numD</name>
+ <type>Number</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>strE</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>intF</name>
+ <type>Integer</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>a1</item>
+ <item>11</item>
+ <item>c1</item>
+ <item/>
+ <item>e1</item>
+ <item>111</item>
+ </line>
+ <line>
+ <item/>
+ <item>22</item>
+ <item/>
+ <item/>
+ <item/>
+ <item/>
+ </line>
+ <line>
+ <item/>
+ <item/>
+ <item>c3</item>
+ <item/>
+ <item/>
+ <item>333</item>
+ </line>
+ <line>
+ <item/>
+ <item>44</item>
+ <item>c4</item>
+ <item>4.4</item>
+ <item/>
+ <item/>
+ </line>
+ <line>
+ <item/>
+ <item/>
+ <item/>
+ <item/>
+ <item>e5</item>
+ <item/>
+ </line>
+ <line>
+ <item>a6</item>
+ <item/>
+ <item/>
+ <item/>
+ <item/>
+ <item>666</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Verify</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>416</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/0024-if-null-specific-fields.hpl
b/integration-tests/transforms/0024-if-null-specific-fields.hpl
new file mode 100644
index 0000000..33c9607
--- /dev/null
+++ b/integration-tests/transforms/0024-if-null-specific-fields.hpl
@@ -0,0 +1,241 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<pipeline>
+ <info>
+ <name>0024-if-null-specific-fields</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2021/06/01 15:47:03.315</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/06/01 15:47:03.315</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Sample data</from>
+ <to>If Null - fields a,c,d,f</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>If Null - fields a,c,d,f</from>
+ <to>Verify</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>If Null - fields a,c,d,f</name>
+ <type>IfNull</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <replaceAllByValue>?</replaceAllByValue>
+ <replaceAllMask/>
+ <selectFields>Y</selectFields>
+ <selectValuesType>N</selectValuesType>
+ <setEmptyStringAll>N</setEmptyStringAll>
+ <valuetypes>
+ </valuetypes>
+ <fields>
+ <field>
+ <name>a</name>
+ <value>a?</value>
+ <mask/>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>c</name>
+ <value>c?</value>
+ <mask/>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>d</name>
+ <value>d?</value>
+ <mask/>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>f</name>
+ <value>f?</value>
+ <mask/>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>272</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Sample data</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>a</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>b</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>c</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>d</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>e</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>f</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>a1</item>
+ <item/>
+ <item>c1</item>
+ <item/>
+ <item>e1</item>
+ <item>f1</item>
+ </line>
+ <line>
+ <item/>
+ <item>b2</item>
+ <item/>
+ <item/>
+ <item/>
+ <item/>
+ </line>
+ <line>
+ <item/>
+ <item/>
+ <item>c3</item>
+ <item/>
+ <item/>
+ <item>f3</item>
+ </line>
+ <line>
+ <item/>
+ <item>b4</item>
+ <item>c4</item>
+ <item>d4</item>
+ <item/>
+ <item/>
+ </line>
+ <line>
+ <item/>
+ <item/>
+ <item/>
+ <item/>
+ <item>e5</item>
+ <item/>
+ </line>
+ <line>
+ <item>a6</item>
+ <item/>
+ <item/>
+ <item/>
+ <item/>
+ <item>f6</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Verify</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>448</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/0023-number-range.hpl
b/integration-tests/transforms/0025-null-if-basic.hpl
similarity index 51%
copy from integration-tests/transforms/0023-number-range.hpl
copy to integration-tests/transforms/0025-null-if-basic.hpl
index afa9515..5e88880 100644
--- a/integration-tests/transforms/0023-number-range.hpl
+++ b/integration-tests/transforms/0025-null-if-basic.hpl
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<pipeline>
<info>
- <name>0023-number-range</name>
+ <name>0025-null-if-basic</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@@ -13,9 +13,9 @@
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
- <created_date>2021/05/31 14:12:11.112</created_date>
+ <created_date>2021/06/01 17:40:11.867</created_date>
<modified_user>-</modified_user>
- <modified_date>2021/05/31 14:12:11.112</modified_date>
+ <modified_date>2021/06/01 17:40:11.867</modified_date>
<key_for_session_key/>
<is_key_private>N</is_key_private>
</info>
@@ -23,18 +23,53 @@
</notepads>
<order>
<hop>
- <from>Prices</from>
- <to>Price range</to>
+ <from>Sample data</from>
+ <to>Null if</to>
<enabled>Y</enabled>
</hop>
<hop>
- <from>Price range</from>
- <to>Verify</to>
+ <from>Null if</from>
+ <to>Preview</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
- <name>Prices</name>
+ <name>Null if</name>
+ <type>NullIf</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>strA</name>
+ <value>a3</value>
+ </field>
+ <field>
+ <name>intB</name>
+ <value>22</value>
+ </field>
+ <field>
+ <name>numC</name>
+ <value>333.33</value>
+ </field>
+ <field>
+ <name>boolD</name>
+ <value>false</value>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Sample data</name>
<type>DataGrid</type>
<description/>
<distribute>Y</distribute>
@@ -46,7 +81,18 @@
</partitioning>
<fields>
<field>
- <name>id</name>
+ <name>strA</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>intB</name>
<type>Integer</type>
<format/>
<currency/>
@@ -57,11 +103,22 @@
<set_empty_string>N</set_empty_string>
</field>
<field>
- <name>price</name>
+ <name>numC</name>
<type>Number</type>
- <format>#.#</format>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>boolD</name>
+ <type>Boolean</type>
+ <format/>
<currency/>
- <decimal>.</decimal>
+ <decimal/>
<group/>
<length>-1</length>
<precision>-1</precision>
@@ -70,101 +127,38 @@
</fields>
<data>
<line>
- <item>1</item>
- <item>9.99</item>
- </line>
- <line>
- <item>2</item>
- <item>6.23</item>
- </line>
- <line>
- <item>3</item>
- <item>57.41</item>
- </line>
- <line>
- <item>4</item>
- <item>299.99</item>
- </line>
- <line>
- <item>5</item>
- <item>100.00</item>
+ <item>a1</item>
+ <item>11</item>
+ <item>111.11</item>
+ <item>true</item>
</line>
<line>
- <item>6</item>
- <item>-103.97</item>
+ <item>a2</item>
+ <item>22</item>
+ <item>222.22</item>
+ <item>false</item>
</line>
<line>
- <item>7</item>
- <item>974.95</item>
+ <item>a3</item>
+ <item>33</item>
+ <item>333.33</item>
+ <item>true</item>
</line>
<line>
- <item>8</item>
- <item>2399.99</item>
- </line>
- <line>
- <item>9</item>
- <item>14.99</item>
- </line>
- <line>
- <item>10</item>
- <item>123.45</item>
+ <item>a4</item>
+ <item>44</item>
+ <item>444.44</item>
+ <item>false</item>
</line>
</data>
<attributes/>
<GUI>
- <xloc>160</xloc>
- <yloc>112</yloc>
- </GUI>
- </transform>
- <transform>
- <name>Price range</name>
- <type>NumberRange</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
- <inputField>price</inputField>
- <outputField>range</outputField>
- <fallBackValue>illegal</fallBackValue>
- <rules>
- <rule>
- <lower_bound>-1.7976931348623157E308</lower_bound>
- <upper_bound>10.0</upper_bound>
- <value>bargain</value>
- </rule>
- <rule>
- <lower_bound>10.0</lower_bound>
- <upper_bound>50.0</upper_bound>
- <value>cheap</value>
- </rule>
- <rule>
- <lower_bound>50.0</lower_bound>
- <upper_bound>250.0</upper_bound>
- <value>average</value>
- </rule>
- <rule>
- <lower_bound>250.0</lower_bound>
- <upper_bound>1000.0</upper_bound>
- <value>pricey</value>
- </rule>
- <rule>
- <lower_bound>1000.0</lower_bound>
- <upper_bound>1.7976931348623157E308</upper_bound>
- <value>expensive</value>
- </rule>
- </rules>
- <attributes/>
- <GUI>
- <xloc>320</xloc>
- <yloc>112</yloc>
+ <xloc>96</xloc>
+ <yloc>128</yloc>
</GUI>
</transform>
<transform>
- <name>Verify</name>
+ <name>Preview</name>
<type>Dummy</type>
<description/>
<distribute>Y</distribute>
@@ -176,8 +170,8 @@
</partitioning>
<attributes/>
<GUI>
- <xloc>480</xloc>
- <yloc>112</yloc>
+ <xloc>416</xloc>
+ <yloc>128</yloc>
</GUI>
</transform>
<transform_error_handling>
diff --git
a/integration-tests/transforms/datasets/golden-if-null-all-fields.csv
b/integration-tests/transforms/datasets/golden-if-null-all-fields.csv
new file mode 100644
index 0000000..d82ab6a
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-if-null-all-fields.csv
@@ -0,0 +1,7 @@
+a,b,c,d,e,f
+a1,?,c1,?,e1,f1
+?,b2,?,?,?,?
+?,?,c3,?,?,f3
+?,b4,c4,d4,?,?
+?,?,?,?,e5,?
+a6,?,?,?,?,f6
diff --git a/integration-tests/transforms/datasets/golden-if-null-by-type.csv
b/integration-tests/transforms/datasets/golden-if-null-by-type.csv
new file mode 100644
index 0000000..c024c4a
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-if-null-by-type.csv
@@ -0,0 +1,7 @@
+strA,intB,strC,numD,strE,intF
+a1,11,c1,0.5,e1,111
+?,22,?,0.5,?,999
+?,999,c3,0.5,?,333
+?,44,c4,4.4,?,999
+?,999,?,0.5,e5,999
+a6,999,?,0.5,?,666
diff --git
a/integration-tests/transforms/datasets/golden-if-null-specific-fields.csv
b/integration-tests/transforms/datasets/golden-if-null-specific-fields.csv
new file mode 100644
index 0000000..bf1d5d3
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-if-null-specific-fields.csv
@@ -0,0 +1,7 @@
+a,b,c,d,e,f
+a1,,c1,d?,e1,f1
+a?,b2,c?,d?,,f?
+a?,,c3,d?,,f3
+a?,b4,c4,d4,,f?
+a?,,c?,d?,e5,f?
+a6,,c?,d?,,f6
diff --git a/integration-tests/transforms/datasets/golden-null-if-basic.csv
b/integration-tests/transforms/datasets/golden-null-if-basic.csv
new file mode 100644
index 0000000..7bd351b
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-null-if-basic.csv
@@ -0,0 +1,5 @@
+strA,intB,numC,boolD
+a1,11,111.11,Y
+a2,,222.22,
+,33,,Y
+a4,44,444.44,
diff --git a/integration-tests/transforms/datasets/golden-number-rang.csv
b/integration-tests/transforms/datasets/golden-number-rang.csv
index 4600a8c..17830fb 100644
--- a/integration-tests/transforms/datasets/golden-number-rang.csv
+++ b/integration-tests/transforms/datasets/golden-number-rang.csv
@@ -1,11 +1,11 @@
id,price,range
-1,10,bargain
-2,6.2,bargain
-3,57.4,average
-4,300,pricey
-5,100,average
-6,-104,bargain
-7,975,pricey
-8,2400,expensive
-9,15,cheap
-10,123.5,average
+1,9.99,bargain
+2,6.23,bargain
+3,57.41,average
+4,299.99,pricey
+5,100.00,average
+6,-103.97,bargain
+7,974.95,pricey
+8,2399.99,expensive
+9,14.99,cheap
+10,123.45,average
diff --git a/integration-tests/transforms/main-0023-number-range.hwf
b/integration-tests/transforms/main-0023-number-range.hwf
new file mode 100644
index 0000000..785e28f
--- /dev/null
+++ b/integration-tests/transforms/main-0023-number-range.hwf
@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<workflow>
+ <name>main-0023-number-range</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2021/05/31 14:18:33.849</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/05/31 14:18:33.849</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <xloc>144</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run Number Range Tests</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0023-number-range UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>352</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Run Number Range Tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/transforms/main-0024-if-null.hwf
b/integration-tests/transforms/main-0024-if-null.hwf
new file mode 100644
index 0000000..690095e
--- /dev/null
+++ b/integration-tests/transforms/main-0024-if-null.hwf
@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<workflow>
+ <name>main-0024-if-null</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2021/06/01 17:45:49.992</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/06/01 17:45:49.992</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <xloc>128</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run If Null Tests</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0024-if-null-specific-fields UNIT</name>
+ </test_name>
+ <test_name>
+ <name>0024-if-null-by-type UNIT</name>
+ </test_name>
+ <test_name>
+ <name>0024-if-null-all-fields UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>336</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Run If Null Tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/transforms/main-0025-null-if.hwf
b/integration-tests/transforms/main-0025-null-if.hwf
new file mode 100644
index 0000000..7e17722
--- /dev/null
+++ b/integration-tests/transforms/main-0025-null-if.hwf
@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<workflow>
+ <name>main-0025-null-if</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2021/06/01 17:45:49.992</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/06/01 17:45:49.992</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <xloc>128</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run Null If Tests</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0025-null-if-basic UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>336</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Run Null If Tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-if-null-all-fields.json
b/integration-tests/transforms/metadata/dataset/golden-if-null-all-fields.json
new file mode 100644
index 0000000..3276b4f
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-if-null-all-fields.json
@@ -0,0 +1,56 @@
+{
+ "base_filename": "golden-if-null-all-fields.csv",
+ "name": "golden-if-null-all-fields",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "a"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "b"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "c"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "d"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "e"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "f"
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/dataset/golden-if-null-by-type.json
b/integration-tests/transforms/metadata/dataset/golden-if-null-by-type.json
new file mode 100644
index 0000000..182a6c1
--- /dev/null
+++ b/integration-tests/transforms/metadata/dataset/golden-if-null-by-type.json
@@ -0,0 +1,56 @@
+{
+ "base_filename": "golden-if-null-by-type.csv",
+ "name": "golden-if-null-by-type",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "strA"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_format": "####0;-####0",
+ "field_name": "intB"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "strC"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 1,
+ "field_precision": -1,
+ "field_format": "####0.0#########;-####0.0#########",
+ "field_name": "numD"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "strE"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_format": "####0;-####0",
+ "field_name": "intF"
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/dataset/golden-if-null-specific-fields.json
b/integration-tests/transforms/metadata/dataset/golden-if-null-specific-fields.json
new file mode 100644
index 0000000..c71d8ad
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-if-null-specific-fields.json
@@ -0,0 +1,56 @@
+{
+ "base_filename": "golden-if-null-specific-fields.csv",
+ "name": "golden-if-null-specific-fields",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "a"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "b"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "c"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "d"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "e"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "f"
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/dataset/golden-number-range.json
b/integration-tests/transforms/metadata/dataset/golden-null-if-basic.json
similarity index 52%
copy from integration-tests/transforms/metadata/dataset/golden-number-range.json
copy to integration-tests/transforms/metadata/dataset/golden-null-if-basic.json
index c8bb503..b2748ef 100644
--- a/integration-tests/transforms/metadata/dataset/golden-number-range.json
+++ b/integration-tests/transforms/metadata/dataset/golden-null-if-basic.json
@@ -1,31 +1,39 @@
{
- "base_filename": "golden-number-rang.csv",
- "name": "golden-number-range",
+ "base_filename": "golden-null-if-basic.csv",
+ "name": "golden-null-if-basic",
"description": "",
"dataset_fields": [
{
"field_comment": "",
"field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "strA"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
"field_type": 5,
"field_precision": 0,
"field_format": "####0;-####0",
- "field_name": "id"
+ "field_name": "intB"
},
{
"field_comment": "",
"field_length": -1,
"field_type": 1,
"field_precision": -1,
- "field_format": "#.#",
- "field_name": "price"
+ "field_format": "####0.0#########;-####0.0#########",
+ "field_name": "numC"
},
{
"field_comment": "",
- "field_length": 255,
- "field_type": 2,
+ "field_length": -1,
+ "field_type": 4,
"field_precision": -1,
"field_format": "",
- "field_name": "range"
+ "field_name": "boolD"
}
],
"folder_name": ""
diff --git
a/integration-tests/transforms/metadata/dataset/golden-number-range.json
b/integration-tests/transforms/metadata/dataset/golden-number-range.json
index c8bb503..0f1c20d 100644
--- a/integration-tests/transforms/metadata/dataset/golden-number-range.json
+++ b/integration-tests/transforms/metadata/dataset/golden-number-range.json
@@ -16,7 +16,7 @@
"field_length": -1,
"field_type": 1,
"field_precision": -1,
- "field_format": "#.#",
+ "field_format": "#.00",
"field_name": "price"
},
{
diff --git a/integration-tests/transforms/metadata/unit-test/0023-number-range
UNIT.json b/integration-tests/transforms/metadata/unit-test/0023-number-range
UNIT.json
index b4ae362..21c75db 100644
--- a/integration-tests/transforms/metadata/unit-test/0023-number-range
UNIT.json
+++ b/integration-tests/transforms/metadata/unit-test/0023-number-range
UNIT.json
@@ -11,16 +11,16 @@
"data_set_field": "id"
},
{
- "transform_field": "price",
- "data_set_field": "price"
- },
- {
"transform_field": "range",
"data_set_field": "range"
+ },
+ {
+ "transform_field": "price",
+ "data_set_field": "price"
}
],
"field_order": [
- ""
+ "id"
],
"transform_name": "Verify",
"data_set_name": "golden-number-range"
diff --git
a/integration-tests/transforms/metadata/unit-test/0024-if-null-all-fields
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0024-if-null-all-fields
UNIT.json
new file mode 100644
index 0000000..f8b1d6e
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0024-if-null-all-fields
UNIT.json
@@ -0,0 +1,53 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": true,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "a",
+ "data_set_field": "a"
+ },
+ {
+ "transform_field": "b",
+ "data_set_field": "b"
+ },
+ {
+ "transform_field": "c",
+ "data_set_field": "c"
+ },
+ {
+ "transform_field": "d",
+ "data_set_field": "d"
+ },
+ {
+ "transform_field": "e",
+ "data_set_field": "e"
+ },
+ {
+ "transform_field": "f",
+ "data_set_field": "f"
+ }
+ ],
+ "field_order": [
+ "a",
+ "b",
+ "c",
+ "d",
+ "e",
+ "f"
+ ],
+ "transform_name": "Verify",
+ "data_set_name": "golden-if-null-all-fields"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0024-if-null-all-fields UNIT",
+ "description": "",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename":
"/home/matt/git/mattcasters/incubator-hop/integration-tests/transforms/0024-if-null-all-fields.hpl",
+ "test_type": "UNIT_TEST"
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/unit-test/0024-if-null-by-type
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0024-if-null-by-type UNIT.json
new file mode 100644
index 0000000..722a2fc
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0024-if-null-by-type
UNIT.json
@@ -0,0 +1,53 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": true,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "intB",
+ "data_set_field": "intB"
+ },
+ {
+ "transform_field": "intF",
+ "data_set_field": "intF"
+ },
+ {
+ "transform_field": "numD",
+ "data_set_field": "numD"
+ },
+ {
+ "transform_field": "strA",
+ "data_set_field": "strA"
+ },
+ {
+ "transform_field": "strC",
+ "data_set_field": "strC"
+ },
+ {
+ "transform_field": "strE",
+ "data_set_field": "strE"
+ }
+ ],
+ "field_order": [
+ "strA",
+ "intB",
+ "strC",
+ "numD",
+ "strE",
+ "intF"
+ ],
+ "transform_name": "Verify",
+ "data_set_name": "golden-if-null-by-type"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0024-if-null-by-type UNIT",
+ "description": "",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename":
"/home/matt/git/mattcasters/incubator-hop/integration-tests/transforms/0024-if-null-by-type.hpl",
+ "test_type": "UNIT_TEST"
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/unit-test/0024-if-null-specific-fields
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0024-if-null-specific-fields
UNIT.json
new file mode 100644
index 0000000..e0cc4f7
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0024-if-null-specific-fields
UNIT.json
@@ -0,0 +1,53 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": true,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "a",
+ "data_set_field": "a"
+ },
+ {
+ "transform_field": "b",
+ "data_set_field": "b"
+ },
+ {
+ "transform_field": "c",
+ "data_set_field": "c"
+ },
+ {
+ "transform_field": "d",
+ "data_set_field": "d"
+ },
+ {
+ "transform_field": "e",
+ "data_set_field": "e"
+ },
+ {
+ "transform_field": "f",
+ "data_set_field": "f"
+ }
+ ],
+ "field_order": [
+ "a",
+ "b",
+ "c",
+ "d",
+ "e",
+ "f"
+ ],
+ "transform_name": "Verify",
+ "data_set_name": "golden-if-null-specific-fields"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0024-if-null-specific-fields UNIT",
+ "description": "",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename":
"/home/matt/git/mattcasters/incubator-hop/integration-tests/transforms/0024-if-null-specific-fields.hpl",
+ "test_type": "UNIT_TEST"
+}
\ No newline at end of file
diff --git a/integration-tests/transforms/metadata/unit-test/0025-null-if-basic
UNIT.json b/integration-tests/transforms/metadata/unit-test/0025-null-if-basic
UNIT.json
new file mode 100644
index 0000000..1179c66
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0025-null-if-basic
UNIT.json
@@ -0,0 +1,43 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": true,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "boolD",
+ "data_set_field": "boolD"
+ },
+ {
+ "transform_field": "intB",
+ "data_set_field": "intB"
+ },
+ {
+ "transform_field": "numC",
+ "data_set_field": "numC"
+ },
+ {
+ "transform_field": "strA",
+ "data_set_field": "strA"
+ }
+ ],
+ "field_order": [
+ "strA",
+ "intB",
+ "numC",
+ "boolD"
+ ],
+ "transform_name": "Preview",
+ "data_set_name": "golden-null-if-basic"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0025-null-if-basic UNIT",
+ "description": "",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename":
"/home/matt/git/mattcasters/incubator-hop/integration-tests/transforms/0025-null-if-basic.hpl",
+ "test_type": "UNIT_TEST"
+}
\ No newline at end of file
