This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new c4f3296878 Improving Data Validation Performance, fixes #7589 (#7812)
c4f3296878 is described below
commit c4f3296878da8484611ff43794bc0a5cf25f5bf7
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Aug 6 22:10:19 2026 +0200
Improving Data Validation Performance, fixes #7589 (#7812)
---
.../ROOT/pages/pipeline/transforms/validator.adoc | 6 +
.../0067-data-validator-source-multicopy.hpl | 380 +++++++++++++++++++++
.../0067-data-validator-source-multifield.hpl | 288 ++++++++++++++++
...lden-data-validator-source-multicopy-errors.csv | 6 +
...lden-data-validator-source-multicopy-output.csv | 6 +
...den-data-validator-source-multifield-errors.csv | 10 +
...den-data-validator-source-multifield-output.csv | 4 +
.../transforms/main-0067-data-validator.hwf | 6 +
...den-data-validator-source-multicopy-errors.json | 56 +++
...den-data-validator-source-multicopy-output.json | 24 ++
...en-data-validator-source-multifield-errors.json | 56 +++
...en-data-validator-source-multifield-output.json | 24 ++
.../0067-data-validator-source-multicopy UNIT.json | 71 ++++
...0067-data-validator-source-multifield UNIT.json | 71 ++++
.../validator/HopValidatorException.java | 10 +
.../pipeline/transforms/validator/Validator.java | 365 +++++++++++++++-----
.../transforms/validator/ValidatorData.java | 31 ++
.../transforms/validator/ValidatorDialog.java | 19 +-
.../transforms/validator/ValidatorMeta.java | 50 +++
.../validator/messages/messages_en_US.properties | 4 +
.../transforms/validator/ValidatorTest.java | 158 +++++++++
21 files changed, 1568 insertions(+), 77 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/validator.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/validator.adoc
index c2aa039543..2887dc1975 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/validator.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/validator.adoc
@@ -60,6 +60,12 @@ Once you select a validation on the left hand side of the
dialog you can specify
|Output one row, concatenate errors with a separator
|If this is enabled error codes and descriptions are concatenated with the
specified separator string.
+|Do not log failed validation data
+|When enabled, the failed field and row values are left out of the validation
error messages and shown as `?` instead. The error messages themselves are
still logged and error rows are still sent to the error handling hop.
+
+|Do not log a line for every rejected row
+|When enabled, the transform no longer writes an error line to the log for
every row it rejects. Error rows are still sent to the error handling hop. On
streams where a large share of the rows is rejected, writing that log line
costs considerably more than the validation itself, so enabling this option can
speed up such pipelines by an order of magnitude.
+
|Validation description
|Optionally you can describe what you're testing for here.
diff --git
a/integration-tests/transforms/0067-data-validator-source-multicopy.hpl
b/integration-tests/transforms/0067-data-validator-source-multicopy.hpl
new file mode 100644
index 0000000000..66f05c4754
--- /dev/null
+++ b/integration-tests/transforms/0067-data-validator-source-multicopy.hpl
@@ -0,0 +1,380 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0067-data-validator-source-multicopy</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2023/03/08 15:11:16.589</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2023/03/08 15:11:16.589</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Validate data</from>
+ <to>sort errors</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Validate data</from>
+ <to>sort output</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>sort output</from>
+ <to>Output</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>sort errors</from>
+ <to>Errors</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>sample data</from>
+ <to>Validate data</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>allowed values</from>
+ <to>Validate data</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Errors</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>496</xloc>
+ <yloc>176</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Output</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>496</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Validate data</name>
+ <type>Validator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>4</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <concat_errors>N</concat_errors>
+ <validate_all>Y</validate_all>
+ <validator_field>
+ <allowed_value>
+</allowed_value>
+ <data_type>String</data_type>
+ <data_type_verified>N</data_type_verified>
+ <error_code>STR1_DICTIONARY</error_code>
+ <error_description>Field string1 is not matching with lookup
values.</error_description>
+ <is_sourcing_values>Y</is_sourcing_values>
+ <max_length/>
+ <min_length/>
+ <name>string1</name>
+ <null_allowed>Y</null_allowed>
+ <only_null_allowed>N</only_null_allowed>
+ <only_numeric_allowed>N</only_numeric_allowed>
+ <sourcing_field>allowed_value</sourcing_field>
+ <sourcing_transform>allowed values</sourcing_transform>
+ <validation_name>string1: verify against allowed values</validation_name>
+ </validator_field>
+ <validator_field>
+ <allowed_value>
+</allowed_value>
+ <data_type>String</data_type>
+ <data_type_verified>N</data_type_verified>
+ <error_code>STR2_DICTIONARY</error_code>
+ <error_description>Field string2 is not matching with lookup
values.</error_description>
+ <is_sourcing_values>Y</is_sourcing_values>
+ <max_length/>
+ <min_length/>
+ <name>string2</name>
+ <null_allowed>Y</null_allowed>
+ <only_null_allowed>N</only_null_allowed>
+ <only_numeric_allowed>N</only_numeric_allowed>
+ <sourcing_field>allowed_value</sourcing_field>
+ <sourcing_transform>allowed values</sourcing_transform>
+ <validation_name>string2: verify against allowed values</validation_name>
+ </validator_field>
+ <attributes/>
+ <GUI>
+ <xloc>304</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>allowed values</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>N</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>A</item>
+ </line>
+ <line>
+ <item>B</item>
+ </line>
+ <line>
+ <item>C</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>allowed_value</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>208</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>sample data</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>A</item>
+ <item>B</item>
+ </line>
+ <line>
+ <item>B</item>
+ <item>X</item>
+ </line>
+ <line>
+ <item>C</item>
+ <item>A</item>
+ </line>
+ <line>
+ <item>X</item>
+ <item>C</item>
+ </line>
+ <line>
+ <item>B</item>
+ <item>Z</item>
+ </line>
+ <line>
+ <item>A</item>
+ <item>A</item>
+ </line>
+ <line>
+ <item>Z</item>
+ <item>B</item>
+ </line>
+ <line>
+ <item>A</item>
+ <item>A</item>
+ </line>
+ <line>
+ <item>B</item>
+ <item>C</item>
+ </line>
+ <line>
+ <item>C</item>
+ <item>Y</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>string1</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>string2</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>sort output</name>
+ <type>SortRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <directory>${java.io.tmpdir}</directory>
+ <prefix>out</prefix>
+ <sort_size>1000000</sort_size>
+ <free_memory/>
+ <compress>N</compress>
+ <compress_variable/>
+ <unique_rows>N</unique_rows>
+ <fields>
+ <field>
+ <name>string1</name>
+ <ascending>Y</ascending>
+ <case_sensitive>Y</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ <field>
+ <name>string2</name>
+ <ascending>Y</ascending>
+ <case_sensitive>Y</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>400</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>sort errors</name>
+ <type>SortRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <directory>${java.io.tmpdir}</directory>
+ <prefix>out</prefix>
+ <sort_size>1000000</sort_size>
+ <free_memory/>
+ <compress>N</compress>
+ <compress_variable/>
+ <unique_rows>N</unique_rows>
+ <fields>
+ <field>
+ <name>string1</name>
+ <ascending>Y</ascending>
+ <case_sensitive>Y</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ <field>
+ <name>string2</name>
+ <ascending>Y</ascending>
+ <case_sensitive>Y</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ <field>
+ <name>errorCodes</name>
+ <ascending>Y</ascending>
+ <case_sensitive>Y</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>400</xloc>
+ <yloc>176</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ <error>
+ <source_transform>Validate data</source_transform>
+ <target_transform>sort errors</target_transform>
+ <is_enabled>Y</is_enabled>
+ <nr_valuename>errorCount</nr_valuename>
+ <descriptions_valuename>errorDescriptions</descriptions_valuename>
+ <fields_valuename>errorFields</fields_valuename>
+ <codes_valuename>errorCodes</codes_valuename>
+ <max_errors/>
+ <max_pct_errors/>
+ <min_pct_rows/>
+ </error>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/transforms/0067-data-validator-source-multifield.hpl
b/integration-tests/transforms/0067-data-validator-source-multifield.hpl
new file mode 100644
index 0000000000..1f77e139ef
--- /dev/null
+++ b/integration-tests/transforms/0067-data-validator-source-multifield.hpl
@@ -0,0 +1,288 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0067-data-validator-source-multifield</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2023/03/08 15:11:16.589</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2023/03/08 15:11:16.589</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Validate data</from>
+ <to>Errors</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Validate data</from>
+ <to>Output</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>sample data</from>
+ <to>Validate data</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>allowed values</from>
+ <to>Validate data</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Errors</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>496</xloc>
+ <yloc>176</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Output</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>496</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Validate data</name>
+ <type>Validator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <concat_errors>N</concat_errors>
+ <validate_all>Y</validate_all>
+ <validator_field>
+ <allowed_value>
+</allowed_value>
+ <data_type>String</data_type>
+ <data_type_verified>N</data_type_verified>
+ <error_code>STR1_DICTIONARY</error_code>
+ <error_description>Field string1 is not matching with lookup
values.</error_description>
+ <is_sourcing_values>Y</is_sourcing_values>
+ <max_length/>
+ <min_length/>
+ <name>string1</name>
+ <null_allowed>Y</null_allowed>
+ <only_null_allowed>N</only_null_allowed>
+ <only_numeric_allowed>N</only_numeric_allowed>
+ <sourcing_field>allowed_value</sourcing_field>
+ <sourcing_transform>allowed values</sourcing_transform>
+ <validation_name>string1: verify against allowed values</validation_name>
+ </validator_field>
+ <validator_field>
+ <allowed_value>
+</allowed_value>
+ <data_type>String</data_type>
+ <data_type_verified>N</data_type_verified>
+ <error_code>STR2_DICTIONARY</error_code>
+ <error_description>Field string2 is not matching with lookup
values.</error_description>
+ <is_sourcing_values>Y</is_sourcing_values>
+ <max_length/>
+ <min_length/>
+ <name>string2</name>
+ <null_allowed>Y</null_allowed>
+ <only_null_allowed>N</only_null_allowed>
+ <only_numeric_allowed>N</only_numeric_allowed>
+ <sourcing_field>allowed_value2</sourcing_field>
+ <sourcing_transform>allowed values</sourcing_transform>
+ <validation_name>string2: verify against allowed values</validation_name>
+ </validator_field>
+ <attributes/>
+ <GUI>
+ <xloc>304</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>allowed values</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>A</item>
+ <item>X</item>
+ </line>
+ <line>
+ <item>B</item>
+ <item>Y</item>
+ </line>
+ <line>
+ <item>C</item>
+ <item>Z</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>allowed_value</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>allowed_value2</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>208</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>sample data</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>A</item>
+ <item>B</item>
+ </line>
+ <line>
+ <item>B</item>
+ <item>X</item>
+ </line>
+ <line>
+ <item>C</item>
+ <item>A</item>
+ </line>
+ <line>
+ <item>X</item>
+ <item>C</item>
+ </line>
+ <line>
+ <item>B</item>
+ <item>Z</item>
+ </line>
+ <line>
+ <item>A</item>
+ <item>A</item>
+ </line>
+ <line>
+ <item>Z</item>
+ <item>B</item>
+ </line>
+ <line>
+ <item>A</item>
+ <item>A</item>
+ </line>
+ <line>
+ <item>B</item>
+ <item>C</item>
+ </line>
+ <line>
+ <item>C</item>
+ <item>Y</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>string1</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>string2</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ <error>
+ <source_transform>Validate data</source_transform>
+ <target_transform>Errors</target_transform>
+ <is_enabled>Y</is_enabled>
+ <nr_valuename>errorCount</nr_valuename>
+ <descriptions_valuename>errorDescriptions</descriptions_valuename>
+ <fields_valuename>errorFields</fields_valuename>
+ <codes_valuename>errorCodes</codes_valuename>
+ <max_errors/>
+ <max_pct_errors/>
+ <min_pct_rows/>
+ </error>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-data-validator-source-multicopy-errors.csv
b/integration-tests/transforms/datasets/golden-data-validator-source-multicopy-errors.csv
new file mode 100644
index 0000000000..ece3cefd05
--- /dev/null
+++
b/integration-tests/transforms/datasets/golden-data-validator-source-multicopy-errors.csv
@@ -0,0 +1,6 @@
+string1,string2,errorCount,errorDescriptions,errorFields,errorCodes
+B,X,1,Field string2 is not matching with lookup values.,string2,STR2_DICTIONARY
+B,Z,1,Field string2 is not matching with lookup values.,string2,STR2_DICTIONARY
+C,Y,1,Field string2 is not matching with lookup values.,string2,STR2_DICTIONARY
+X,C,1,Field string1 is not matching with lookup values.,string1,STR1_DICTIONARY
+Z,B,1,Field string1 is not matching with lookup values.,string1,STR1_DICTIONARY
diff --git
a/integration-tests/transforms/datasets/golden-data-validator-source-multicopy-output.csv
b/integration-tests/transforms/datasets/golden-data-validator-source-multicopy-output.csv
new file mode 100644
index 0000000000..1dae1cc4c1
--- /dev/null
+++
b/integration-tests/transforms/datasets/golden-data-validator-source-multicopy-output.csv
@@ -0,0 +1,6 @@
+string1,string2
+A,A
+A,A
+A,B
+B,C
+C,A
diff --git
a/integration-tests/transforms/datasets/golden-data-validator-source-multifield-errors.csv
b/integration-tests/transforms/datasets/golden-data-validator-source-multifield-errors.csv
new file mode 100644
index 0000000000..d9837b6162
--- /dev/null
+++
b/integration-tests/transforms/datasets/golden-data-validator-source-multifield-errors.csv
@@ -0,0 +1,10 @@
+string1,string2,errorCount,errorDescriptions,errorFields,errorCodes
+A,B,1,Field string2 is not matching with lookup values.,string2,STR2_DICTIONARY
+C,A,1,Field string2 is not matching with lookup values.,string2,STR2_DICTIONARY
+X,C,1,Field string1 is not matching with lookup values.,string1,STR1_DICTIONARY
+X,C,1,Field string2 is not matching with lookup values.,string2,STR2_DICTIONARY
+A,A,1,Field string2 is not matching with lookup values.,string2,STR2_DICTIONARY
+Z,B,1,Field string1 is not matching with lookup values.,string1,STR1_DICTIONARY
+Z,B,1,Field string2 is not matching with lookup values.,string2,STR2_DICTIONARY
+A,A,1,Field string2 is not matching with lookup values.,string2,STR2_DICTIONARY
+B,C,1,Field string2 is not matching with lookup values.,string2,STR2_DICTIONARY
diff --git
a/integration-tests/transforms/datasets/golden-data-validator-source-multifield-output.csv
b/integration-tests/transforms/datasets/golden-data-validator-source-multifield-output.csv
new file mode 100644
index 0000000000..9851f785f0
--- /dev/null
+++
b/integration-tests/transforms/datasets/golden-data-validator-source-multifield-output.csv
@@ -0,0 +1,4 @@
+string1,string2
+B,X
+B,Z
+C,Y
diff --git a/integration-tests/transforms/main-0067-data-validator.hwf
b/integration-tests/transforms/main-0067-data-validator.hwf
index 4d279f6698..b2354cbf4d 100644
--- a/integration-tests/transforms/main-0067-data-validator.hwf
+++ b/integration-tests/transforms/main-0067-data-validator.hwf
@@ -66,6 +66,12 @@ limitations under the License.
<test_name>
<name>0067-data-validator-source UNIT</name>
</test_name>
+ <test_name>
+ <name>0067-data-validator-source-multifield UNIT</name>
+ </test_name>
+ <test_name>
+ <name>0067-data-validator-source-multicopy UNIT</name>
+ </test_name>
</test_names>
<parallel>N</parallel>
<xloc>336</xloc>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-data-validator-source-multicopy-errors.json
b/integration-tests/transforms/metadata/dataset/golden-data-validator-source-multicopy-errors.json
new file mode 100644
index 0000000000..8d9a777a91
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-data-validator-source-multicopy-errors.json
@@ -0,0 +1,56 @@
+{
+ "base_filename": "golden-data-validator-source-multicopy-errors.csv",
+ "name": "golden-data-validator-source-multicopy-errors",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "string1",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "string2",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": 3,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "errorCount",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "errorDescriptions",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "errorFields",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "errorCodes",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
diff --git
a/integration-tests/transforms/metadata/dataset/golden-data-validator-source-multicopy-output.json
b/integration-tests/transforms/metadata/dataset/golden-data-validator-source-multicopy-output.json
new file mode 100644
index 0000000000..8302103e94
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-data-validator-source-multicopy-output.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "golden-data-validator-source-multicopy-output.csv",
+ "name": "golden-data-validator-source-multicopy-output",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "string1",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "string2",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
diff --git
a/integration-tests/transforms/metadata/dataset/golden-data-validator-source-multifield-errors.json
b/integration-tests/transforms/metadata/dataset/golden-data-validator-source-multifield-errors.json
new file mode 100644
index 0000000000..04ed64a7a1
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-data-validator-source-multifield-errors.json
@@ -0,0 +1,56 @@
+{
+ "base_filename": "golden-data-validator-source-multifield-errors.csv",
+ "name": "golden-data-validator-source-multifield-errors",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "string1",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "string2",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": 3,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "errorCount",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "errorDescriptions",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "errorFields",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "errorCodes",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
diff --git
a/integration-tests/transforms/metadata/dataset/golden-data-validator-source-multifield-output.json
b/integration-tests/transforms/metadata/dataset/golden-data-validator-source-multifield-output.json
new file mode 100644
index 0000000000..dac0bc271f
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-data-validator-source-multifield-output.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "golden-data-validator-source-multifield-output.csv",
+ "name": "golden-data-validator-source-multifield-output",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "string1",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "string2",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
diff --git
a/integration-tests/transforms/metadata/unit-test/0067-data-validator-source-multicopy
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0067-data-validator-source-multicopy
UNIT.json
new file mode 100644
index 0000000000..fe2db6addc
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0067-data-validator-source-multicopy
UNIT.json
@@ -0,0 +1,71 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": true,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "string1",
+ "data_set_field": "string1"
+ },
+ {
+ "transform_field": "string2",
+ "data_set_field": "string2"
+ }
+ ],
+ "field_order": [
+ "string1",
+ "string2"
+ ],
+ "data_set_name": "golden-data-validator-source-multicopy-output",
+ "transform_name": "Output"
+ },
+ {
+ "field_mappings": [
+ {
+ "transform_field": "string1",
+ "data_set_field": "string1"
+ },
+ {
+ "transform_field": "string2",
+ "data_set_field": "string2"
+ },
+ {
+ "transform_field": "errorCount",
+ "data_set_field": "errorCount"
+ },
+ {
+ "transform_field": "errorDescriptions",
+ "data_set_field": "errorDescriptions"
+ },
+ {
+ "transform_field": "errorFields",
+ "data_set_field": "errorFields"
+ },
+ {
+ "transform_field": "errorCodes",
+ "data_set_field": "errorCodes"
+ }
+ ],
+ "field_order": [
+ "string1",
+ "string2",
+ "errorCount",
+ "errorDescriptions",
+ "errorFields",
+ "errorCodes"
+ ],
+ "data_set_name": "golden-data-validator-source-multicopy-errors",
+ "transform_name": "Errors"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0067-data-validator-source-multicopy UNIT",
+ "description": "",
+ "persist_filename": "",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0067-data-validator-source-multicopy.hpl",
+ "test_type": "UNIT_TEST"
+}
diff --git
a/integration-tests/transforms/metadata/unit-test/0067-data-validator-source-multifield
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0067-data-validator-source-multifield
UNIT.json
new file mode 100644
index 0000000000..8aa8b624f8
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0067-data-validator-source-multifield
UNIT.json
@@ -0,0 +1,71 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": true,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "string1",
+ "data_set_field": "string1"
+ },
+ {
+ "transform_field": "string2",
+ "data_set_field": "string2"
+ }
+ ],
+ "field_order": [
+ "string1",
+ "string2"
+ ],
+ "data_set_name": "golden-data-validator-source-multifield-output",
+ "transform_name": "Output"
+ },
+ {
+ "field_mappings": [
+ {
+ "transform_field": "string1",
+ "data_set_field": "string1"
+ },
+ {
+ "transform_field": "string2",
+ "data_set_field": "string2"
+ },
+ {
+ "transform_field": "errorCount",
+ "data_set_field": "errorCount"
+ },
+ {
+ "transform_field": "errorDescriptions",
+ "data_set_field": "errorDescriptions"
+ },
+ {
+ "transform_field": "errorFields",
+ "data_set_field": "errorFields"
+ },
+ {
+ "transform_field": "errorCodes",
+ "data_set_field": "errorCodes"
+ }
+ ],
+ "field_order": [
+ "string1",
+ "string2",
+ "errorCount",
+ "errorDescriptions",
+ "errorFields",
+ "errorCodes"
+ ],
+ "data_set_name": "golden-data-validator-source-multifield-errors",
+ "transform_name": "Errors"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0067-data-validator-source-multifield UNIT",
+ "description": "",
+ "persist_filename": "",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0067-data-validator-source-multifield.hpl",
+ "test_type": "UNIT_TEST"
+}
diff --git
a/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/HopValidatorException.java
b/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/HopValidatorException.java
index 0587b8ef59..440e9d6cf6 100644
---
a/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/HopValidatorException.java
+++
b/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/HopValidatorException.java
@@ -74,6 +74,16 @@ public class HopValidatorException extends HopValueException
{
this.fieldName = fieldName;
}
+ /**
+ * This exception carries the outcome of a validation rule to the error
handling hop, it is never
+ * used to debug a code path. Capturing a stack trace for it would cost a
stack walk for every
+ * rejected row, which is pure overhead on a transform whose job is to
reject rows.
+ */
+ @Override
+ public synchronized Throwable fillInStackTrace() {
+ return this;
+ }
+
/**
* @return the code
*/
diff --git
a/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/Validator.java
b/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/Validator.java
index 191fdcc4c8..4d1028375f 100644
---
a/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/Validator.java
+++
b/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/Validator.java
@@ -19,9 +19,12 @@
package org.apache.hop.pipeline.transforms.validator;
import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
@@ -179,15 +182,61 @@ public class Validator extends
BaseTransform<ValidatorMeta, ValidatorData> imple
}
}
+ /**
+ * Every copy of this transform builds its own list of allowed values from
the info transforms, so
+ * every copy needs to receive all the rows of those transforms. That only
happens when the info
+ * transform copies its rows to all target copies. With the default round
robin data movement each
+ * copy would see a different slice of the allowed values and reject
perfectly valid rows, so we
+ * refuse to run instead of silently producing wrong results.
+ *
+ * <p>Partitioned transforms are left alone: there the copies are matched
one to one with the
+ * copies of the info transform, which is a deliberate setup.
+ *
+ * @return true when the allowed values reach every copy of this transform
+ */
+ private boolean verifyInfoTransformsReachEveryCopy() {
+ if (getTransformMeta().isPartitioned() ||
getTransformMeta().getCopies(this) <= 1) {
+ return true;
+ }
+
+ boolean ok = true;
+ List<IStream> streams = meta.getTransformIOMeta().getInfoStreams();
+ for (int i = 0; i < meta.getValidations().size(); i++) {
+ Validation field = meta.getValidations().get(i);
+ if (!field.isSourcingValues() || i >= streams.size()) {
+ continue;
+ }
+ TransformMeta sourceTransform = streams.get(i).getTransformMeta();
+ if (sourceTransform == null || !sourceTransform.isDistributes()) {
+ continue;
+ }
+ logError(
+ BaseMessages.getString(
+ PKG,
+ "Validator.Exception.InfoTransformIsDistributing",
+ sourceTransform.getName(),
+ getTransformName(),
+ Integer.toString(getTransformMeta().getCopies(this)),
+ Const.NVL(field.getName(), field.getFieldName())));
+ ok = false;
+ }
+ return ok;
+ }
+
private void readSourceValuesFromInfoTransforms() throws
HopTransformException {
// The input and the output are the same but without the "info" source
transforms.
//
data.inputRowMeta = getPipelineMeta().getTransformFields(variables,
getTransformName());
- Map<String, Integer> inputTransformWasProcessed = new HashMap<>();
+ List<IStream> streams = meta.getTransformIOMeta().getInfoStreams();
+
+ // Group the validations that source their allowed values by source
transform. An info stream
+ // can only be drained once, so validations reading from the same
transform have to be served
+ // from a single pass over that stream. They can still read different
fields from it.
+ //
+ Map<String, List<Integer>> validationsPerTransform = new LinkedHashMap<>();
for (int i = 0; i < meta.getValidations().size(); i++) {
Validation field = meta.getValidations().get(i);
- List<IStream> streams = meta.getTransformIOMeta().getInfoStreams();
// If we need to source the "allowed values" data from a different
transform, we do this here
// as well.
@@ -206,50 +255,140 @@ public class Validator extends
BaseTransform<ValidatorMeta, ValidatorData> imple
+ "]");
}
- // Still here : OK, read the data from the specified transform.
- // The data is stored in data.listValues[i] and data.constantsMeta.
- //
- String transformName = streams.get(i).getTransformName();
- if (inputTransformWasProcessed.containsKey(transformName)) {
- // The transform was processed for other StreamInterface.
- //
- data.listValues[i] =
data.listValues[inputTransformWasProcessed.get(transformName)];
- data.constantsMeta[i] =
data.constantsMeta[inputTransformWasProcessed.get(transformName)];
- continue;
- }
- IRowSet allowedRowSet = findInputRowSet(transformName);
- int fieldIndex = -1;
- List<Object> allowedValues = new ArrayList<>();
- Object[] allowedRowData = getRowFrom(allowedRowSet);
- while (allowedRowData != null) {
- IRowMeta allowedRowMeta = allowedRowSet.getRowMeta();
- if (fieldIndex < 0) {
- fieldIndex = allowedRowMeta.indexOfValue(field.getSourcingField());
- if (fieldIndex < 0) {
- throw new HopTransformException(
- "Source field ["
- + field.getSourcingField()
- + "] is not found in the source row data");
- }
- data.constantsMeta[i] = allowedRowMeta.getValueMeta(fieldIndex);
- }
- Object allowedValue = allowedRowData[fieldIndex];
- if (allowedValue != null) {
- allowedValues.add(allowedValue);
- }
+ validationsPerTransform
+ .computeIfAbsent(streams.get(i).getTransformName(), name -> new
ArrayList<>())
+ .add(i);
+ }
+ }
+
+ for (Map.Entry<String, List<Integer>> entry :
validationsPerTransform.entrySet()) {
+ readSourceValuesFromInfoTransform(entry.getKey(), entry.getValue());
+ }
+ }
+
+ /**
+ * Read the allowed values of all the validations that source their data
from the given transform.
+ * The stream is read once, the values are stored in data.listValues and
data.constantsMeta.
+ *
+ * @param transformName the info transform to read the allowed values from
+ * @param validationIndexes the validations served by this info transform
+ */
+ private void readSourceValuesFromInfoTransform(
+ String transformName, List<Integer> validationIndexes) throws
HopTransformException {
+ IRowSet allowedRowSet = findInputRowSet(transformName);
+ if (allowedRowSet == null) {
+ throw new HopTransformException(
+ "Unable to find the rows with allowed values coming from transform ["
+ + transformName
+ + "]. Please verify that the hop between ["
+ + transformName
+ + "] and ["
+ + getTransformName()
+ + "] is enabled.");
+ }
- // Grab another row too...
- //
- allowedRowData = getRowFrom(allowedRowSet);
+ int nrValidations = validationIndexes.size();
+ int[] sourceIndexes = new int[nrValidations];
+ Arrays.fill(sourceIndexes, -1);
+ List<List<Object>> allowedValues = new ArrayList<>(nrValidations);
+ for (int v = 0; v < nrValidations; v++) {
+ allowedValues.add(new ArrayList<>());
+ }
+
+ Object[] allowedRowData = getRowFrom(allowedRowSet);
+ while (allowedRowData != null) {
+ IRowMeta allowedRowMeta = allowedRowSet.getRowMeta();
+ for (int v = 0; v < nrValidations; v++) {
+ if (sourceIndexes[v] < 0) {
+ Validation field =
meta.getValidations().get(validationIndexes.get(v));
+ sourceIndexes[v] =
allowedRowMeta.indexOfValue(field.getSourcingField());
+ if (sourceIndexes[v] < 0) {
+ throw new HopTransformException(
+ "Source field ["
+ + field.getSourcingField()
+ + "] is not found in the source row data");
+ }
+ data.constantsMeta[validationIndexes.get(v)] =
+ allowedRowMeta.getValueMeta(sourceIndexes[v]);
+ }
+ Object allowedValue = allowedRowData[sourceIndexes[v]];
+ if (allowedValue != null) {
+ allowedValues.get(v).add(allowedValue);
}
- // Set the list values in the data block...
- //
- data.listValues[i] = allowedValues.toArray(new Object[0]);
- inputTransformWasProcessed.put(transformName, i);
}
+
+ // Grab another row too...
+ //
+ allowedRowData = getRowFrom(allowedRowSet);
+ }
+
+ // Set the list values in the data block...
+ //
+ for (int v = 0; v < nrValidations; v++) {
+ int i = validationIndexes.get(v);
+ data.listValues[i] = allowedValues.get(v).toArray(new Object[0]);
+ data.listValuesSet[i] = createLookupSet(data.constantsMeta[i],
data.listValues[i]);
}
}
+ /**
+ * Build a hash based view on the allowed values so we don't have to walk
the whole list for every
+ * row. This is only safe for data types where two values are equal exactly
when they compare as
+ * equal, so other types simply keep using the linear scan.
+ *
+ * @param constantsMeta the value metadata the allowed values are stored in
+ * @param listValues the allowed values
+ * @return a set of the allowed values, or null when the type doesn't allow
this optimisation
+ */
+ private static Set<Object> createLookupSet(IValueMeta constantsMeta,
Object[] listValues) {
+ if (constantsMeta == null
+ || listValues.length == 0
+ || constantsMeta.getStorageType() != IValueMeta.STORAGE_TYPE_NORMAL
+ || !isHashableType(constantsMeta.getType())) {
+ return null;
+ }
+ Set<Object> set = new HashSet<>((int) (listValues.length / 0.75f) + 1);
+ for (Object value : listValues) {
+ // Null values never match: the validation skips null values before the
list is consulted.
+ if (value != null) {
+ set.add(value);
+ }
+ }
+ return set;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Set<Object>[] newSetArray(int size) {
+ return new Set[size];
+ }
+
+ /**
+ * Types for which equals() and compare()==0 agree. Doubles (-0.0, NaN) and
BigDecimals (scale)
+ * are deliberately left out, and so are dates because they can be mixed
with timestamps.
+ */
+ private static boolean isHashableType(int type) {
+ return type == IValueMeta.TYPE_STRING
+ || type == IValueMeta.TYPE_INTEGER
+ || type == IValueMeta.TYPE_BOOLEAN;
+ }
+
+ /**
+ * The hash lookup is only equivalent to the linear scan when the incoming
value is compared with
+ * plain equality. A different data type, a non normal storage type or a
string that is compared
+ * with a collator, case insensitively or while ignoring whitespace all
change the outcome, so in
+ * those cases we fall back to comparing value by value.
+ */
+ private static boolean canUseLookupSet(IValueMeta valueMeta, IValueMeta
constantsMeta) {
+ if (valueMeta.getType() != constantsMeta.getType()
+ || valueMeta.getStorageType() != IValueMeta.STORAGE_TYPE_NORMAL) {
+ return false;
+ }
+ return valueMeta.getType() != IValueMeta.TYPE_STRING
+ || (valueMeta.isCollatorDisabled()
+ && !valueMeta.isCaseInsensitive()
+ && !valueMeta.isIgnoreWhitespace());
+ }
+
/**
* @param inputRowMeta the input row metadata
* @param r the input row (data)
@@ -258,7 +397,9 @@ public class Validator extends BaseTransform<ValidatorMeta,
ValidatorData> imple
*/
private List<HopValidatorException> validateFields(IRowMeta inputRowMeta,
Object[] r)
throws HopValueException {
- List<HopValidatorException> exceptions = new ArrayList<>();
+ // Reused per row: processRow is done with the list before the next row
comes in.
+ List<HopValidatorException> exceptions = data.exceptions;
+ exceptions.clear();
for (int i = 0; i < meta.getValidations().size(); i++) {
Validation field = meta.getValidations().get(i);
@@ -308,7 +449,7 @@ public class Validator extends BaseTransform<ValidatorMeta,
ValidatorData> imple
}
}
- int dataType = ValueMetaFactory.getIdForValueMeta(field.getDataType());
+ int dataType = data.dataTypes[i];
// Check the data type!
// Same data type?
@@ -340,21 +481,13 @@ public class Validator extends
BaseTransform<ValidatorMeta, ValidatorData> imple
continue;
}
- if (data.fieldsMinimumLengthAsInt[i] >= 0
- || data.fieldsMaximumLengthAsInt[i] >= 0
- || data.minimumValue[i] != null
- || data.maximumValue[i] != null
- || data.listValues[i].length > 0
- || field.isSourcingValues()
- || StringUtils.isNotEmpty(data.startString[i])
- || StringUtils.isNotEmpty(data.endString[i])
- || StringUtils.isNotEmpty(data.startStringNotAllowed[i])
- || StringUtils.isNotEmpty(data.endStringNotAllowed[i])
- || field.isOnlyNumericAllowed()
- || data.patternExpected[i] != null
- || data.patternDisallowed[i] != null) {
-
- String stringValue = valueMeta.getString(valueData);
+ if (data.hasValueChecks[i]) {
+
+ // Only render the value as a string when one of the configured rules
actually needs it.
+ // The placeholder below is never looked at: data.needsStringValue
covers exactly the rules
+ // that use stringValue or stringLength, so keep the two in sync when
adding a rule.
+ //
+ String stringValue = data.needsStringValue[i] ?
valueMeta.getString(valueData) : "";
int stringLength = stringValue.length();
// Minimum length
@@ -450,15 +583,7 @@ public class Validator extends
BaseTransform<ValidatorMeta, ValidatorData> imple
// In list?
//
if (field.isSourcingValues() || data.listValues[i].length > 0) {
- boolean found = false;
- for (Object object : data.listValues[i]) {
- if (object != null
- && data.listValues[i] != null
- && valueMeta.compare(valueData, validatorMeta, object) == 0) {
- found = true;
- }
- }
- if (!found) {
+ if (!isInAllowedValues(i, valueMeta, validatorMeta, valueData)) {
HopValidatorException exception =
new HopValidatorException(
this,
@@ -479,7 +604,9 @@ public class Validator extends BaseTransform<ValidatorMeta,
ValidatorData> imple
// Numeric data or strings with only
if (field.isOnlyNumericAllowed()) {
- HopValidatorException exception = assertNumeric(valueMeta,
valueData, field);
+ HopValidatorException exception =
+ assertNumeric(
+ valueMeta, valueData, field, data.needsStringValue[i] ?
stringValue : null);
if (exception != null) {
exceptions.add(exception);
if (!meta.isValidatingAll()) {
@@ -577,9 +704,8 @@ public class Validator extends BaseTransform<ValidatorMeta,
ValidatorData> imple
// Matching regular expression allowed?
//
- if (data.patternExpected[i] != null) {
- Matcher matcher = data.patternExpected[i].matcher(stringValue);
- if (!matcher.matches()) {
+ if (data.matcherExpected[i] != null) {
+ if (!data.matcherExpected[i].reset(stringValue).matches()) {
HopValidatorException exception =
new HopValidatorException(
this,
@@ -601,9 +727,8 @@ public class Validator extends BaseTransform<ValidatorMeta,
ValidatorData> imple
// Matching regular expression NOT allowed?
//
- if (data.patternDisallowed[i] != null) {
- Matcher matcher = data.patternDisallowed[i].matcher(stringValue);
- if (matcher.matches()) {
+ if (data.matcherDisallowed[i] != null) {
+ if (data.matcherDisallowed[i].reset(stringValue).matches()) {
HopValidatorException exception =
new HopValidatorException(
this,
@@ -628,6 +753,40 @@ public class Validator extends
BaseTransform<ValidatorMeta, ValidatorData> imple
return exceptions;
}
+ /**
+ * Look the value up in the allowed values of validation i. Uses the hash
based lookup when the
+ * data type allows it and falls back to comparing the value with every
allowed value otherwise.
+ *
+ * @param i the index of the validation
+ * @param valueMeta the metadata of the incoming value
+ * @param constantsMeta the metadata the allowed values are stored in
+ * @param valueData the incoming value
+ * @return true if the value is one of the allowed values
+ */
+ private boolean isInAllowedValues(
+ int i, IValueMeta valueMeta, IValueMeta constantsMeta, Object valueData)
+ throws HopValueException {
+ if (data.listValuesSet[i] != null && canUseLookupSet(valueMeta,
constantsMeta)) {
+ return data.listValuesSet[i].contains(valueData);
+ }
+ for (Object object : data.listValues[i]) {
+ if (object != null && valueMeta.compare(valueData, constantsMeta,
object) == 0) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * The engine writes a log line for every rejected row. On a transform whose
job is to reject rows
+ * that log line costs far more than the validation itself, so it can be
switched off while error
+ * rows keep flowing to the error handling hop.
+ */
+ @Override
+ protected boolean isLoggingErrorDescriptions() {
+ return !meta.isSuppressingErrorLog();
+ }
+
/**
* Returns the field value for inclusion in validation messages, or a
placeholder when logging of
* failed data is suppressed.
@@ -653,7 +812,18 @@ public class Validator extends
BaseTransform<ValidatorMeta, ValidatorData> imple
// package-local visibility for testing purposes
HopValidatorException assertNumeric(IValueMeta valueMeta, Object valueData,
Validation field)
throws HopValueException {
- if (valueMeta.isNumeric() ||
containsOnlyDigits(valueMeta.getString(valueData))) {
+ return assertNumeric(valueMeta, valueData, field, null);
+ }
+
+ /**
+ * @param stringValue the value already rendered as a string, or null when
it still has to be
+ * converted. Numeric fields never need the conversion at all.
+ */
+ private HopValidatorException assertNumeric(
+ IValueMeta valueMeta, Object valueData, Validation field, String
stringValue)
+ throws HopValueException {
+ if (valueMeta.isNumeric()
+ || containsOnlyDigits(stringValue != null ? stringValue :
valueMeta.getString(valueData))) {
return null;
}
return new HopValidatorException(
@@ -670,7 +840,8 @@ public class Validator extends BaseTransform<ValidatorMeta,
ValidatorData> imple
}
private boolean containsOnlyDigits(String string) {
- for (char c : string.toCharArray()) {
+ for (int i = 0; i < string.length(); i++) {
+ char c = string.charAt(i);
if (c < '0' || c > '9') {
return false;
}
@@ -695,8 +866,16 @@ public class Validator extends
BaseTransform<ValidatorMeta, ValidatorData> imple
}
meta.searchInfoAndTargetTransforms(transforms);
+ if (!verifyInfoTransformsReachEveryCopy()) {
+ return false;
+ }
+
// initialize arrays of validation data
data.constantsMeta = new IValueMeta[meta.getValidations().size()];
+ data.dataTypes = new int[meta.getValidations().size()];
+ data.listValuesSet = newSetArray(meta.getValidations().size());
+ data.needsStringValue = new boolean[meta.getValidations().size()];
+ data.hasValueChecks = new boolean[meta.getValidations().size()];
data.minimumValueAsString = new String[meta.getValidations().size()];
data.maximumValueAsString = new String[meta.getValidations().size()];
data.fieldsMinimumLengthAsInt = new int[meta.getValidations().size()];
@@ -719,10 +898,15 @@ public class Validator extends
BaseTransform<ValidatorMeta, ValidatorData> imple
data.regularExpressionNotAllowed = new
String[meta.getValidations().size()];
data.patternExpected = new Pattern[meta.getValidations().size()];
data.patternDisallowed = new Pattern[meta.getValidations().size()];
+ data.matcherExpected = new Matcher[meta.getValidations().size()];
+ data.matcherDisallowed = new Matcher[meta.getValidations().size()];
+ data.exceptions = new ArrayList<>();
for (int i = 0; i < meta.getValidations().size(); i++) {
Validation field = meta.getValidations().get(i);
+ // Resolving the data type goes over the plugin registry, so do it once
instead of per row.
int dataType = ValueMetaFactory.getIdForValueMeta(field.getDataType());
+ data.dataTypes[i] = dataType;
try {
initBasics(i, field);
@@ -743,15 +927,44 @@ public class Validator extends
BaseTransform<ValidatorMeta, ValidatorData> imple
if (StringUtils.isNotEmpty(data.regularExpression[i])) {
data.patternExpected[i] = Pattern.compile(data.regularExpression[i]);
+ data.matcherExpected[i] = data.patternExpected[i].matcher("");
}
if (StringUtils.isNotEmpty(data.regularExpressionNotAllowed[i])) {
data.patternDisallowed[i] =
Pattern.compile(data.regularExpressionNotAllowed[i]);
+ data.matcherDisallowed[i] = data.patternDisallowed[i].matcher("");
}
+
+ initRuleFlags(i, field);
+ data.listValuesSet[i] = createLookupSet(data.constantsMeta[i],
data.listValues[i]);
}
return true;
}
+ /**
+ * Work out once which groups of rules are configured for a validation, so
that the row loop
+ * doesn't have to test every single option again for every row.
+ */
+ private void initRuleFlags(int i, Validation field) {
+ data.needsStringValue[i] =
+ data.fieldsMinimumLengthAsInt[i] >= 0
+ || data.fieldsMaximumLengthAsInt[i] >= 0
+ || StringUtils.isNotEmpty(data.startString[i])
+ || StringUtils.isNotEmpty(data.endString[i])
+ || StringUtils.isNotEmpty(data.startStringNotAllowed[i])
+ || StringUtils.isNotEmpty(data.endStringNotAllowed[i])
+ || data.patternExpected[i] != null
+ || data.patternDisallowed[i] != null;
+
+ data.hasValueChecks[i] =
+ data.needsStringValue[i]
+ || data.minimumValue[i] != null
+ || data.maximumValue[i] != null
+ || data.listValues[i].length > 0
+ || field.isSourcingValues()
+ || field.isOnlyNumericAllowed();
+ }
+
private void initBasics(int i, Validation field) throws HopPluginException {
data.constantsMeta[i] = createValueMeta(field.getFieldName(),
field.getDataType());
data.constantsMeta[i].setConversionMask(field.getConversionMask());
diff --git
a/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorData.java
b/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorData.java
index d4f6761f68..2316a8e739 100644
---
a/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorData.java
+++
b/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorData.java
@@ -18,6 +18,9 @@
package org.apache.hop.pipeline.transforms.validator;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
@@ -30,16 +33,44 @@ public class ValidatorData extends BaseTransformData
implements ITransformData {
public int[] fieldIndexes;
public IValueMeta[] constantsMeta;
+
+ /** The data type id of every validation, resolved once so we don't hit the
registry per row. */
+ public int[] dataTypes;
+
public String[] minimumValueAsString;
public String[] maximumValueAsString;
public int[] fieldsMinimumLengthAsInt;
public int[] fieldsMaximumLengthAsInt;
public Object[][] listValues;
+ /**
+ * Hash based view on {@link #listValues}, used to look up allowed values in
constant time. Only
+ * filled in for data types where equality and comparison agree, and only
used when the incoming
+ * value meta compares with plain equality as well. Null means: fall back to
the linear scan.
+ */
+ public Set<Object>[] listValuesSet;
+
+ /** True when at least one validation rule of this field needs the value as
a string. */
+ public boolean[] needsStringValue;
+
+ /** True when the field has any rule that is checked on a non-null value. */
+ public boolean[] hasValueChecks;
+
public Pattern[] patternExpected;
public Pattern[] patternDisallowed;
+ /**
+ * Matchers reused for every row instead of allocating a fresh one per row.
A matcher is stateful,
+ * but this data object belongs to a single transform copy so it is never
shared between threads.
+ */
+ public Matcher[] matcherExpected;
+
+ public Matcher[] matcherDisallowed;
+
+ /** Reused across rows so a clean row doesn't allocate a list to report
nothing in. */
+ public List<HopValidatorException> exceptions;
+
public String[] errorCode;
public String[] errorDescription;
public String[] conversionMask;
diff --git
a/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorDialog.java
b/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorDialog.java
index 861318cb15..57d6b076ce 100644
---
a/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorDialog.java
+++
b/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorDialog.java
@@ -114,6 +114,7 @@ public class ValidatorDialog extends BaseTransformDialog {
private Button wConcatErrors;
private TextVar wConcatSeparator;
private Button wSuppressLogFailedData;
+ private Button wSuppressErrorLog;
public ValidatorDialog(
Shell parent, IVariables variables, ValidatorMeta transformMeta,
PipelineMeta pipelineMeta) {
@@ -222,6 +223,20 @@ public class ValidatorDialog extends BaseTransformDialog {
fdSuppressLogFailedData.top = new FormAttachment(wConcatErrors, margin);
wSuppressLogFailedData.setLayoutData(fdSuppressLogFailedData);
+ // Optionally suppress the log line the engine writes for every rejected
row
+ //
+ wSuppressErrorLog = new Button(shell, SWT.CHECK);
+ wSuppressErrorLog.setText(
+ BaseMessages.getString(PKG, "ValidatorDialog.SuppressErrorLog.Label"));
+ wSuppressErrorLog.setToolTipText(
+ BaseMessages.getString(PKG,
"ValidatorDialog.SuppressErrorLog.Tooltip"));
+ PropsUi.setLook(wSuppressErrorLog);
+ FormData fdSuppressErrorLog = new FormData();
+ fdSuppressErrorLog.left = new FormAttachment(middle, 0);
+ fdSuppressErrorLog.right = new FormAttachment(100, 0);
+ fdSuppressErrorLog.top = new FormAttachment(wSuppressLogFailedData,
margin);
+ wSuppressErrorLog.setLayoutData(fdSuppressErrorLog);
+
// Create a scrolled composite on the right side...
//
ScrolledComposite wSComp = new ScrolledComposite(shell, SWT.H_SCROLL |
SWT.V_SCROLL);
@@ -229,7 +244,7 @@ public class ValidatorDialog extends BaseTransformDialog {
wSComp.setLayout(new FillLayout());
FormData fdComp = new FormData();
fdComp.left = new FormAttachment(middle / 2, margin);
- fdComp.top = new FormAttachment(wSuppressLogFailedData, margin);
+ fdComp.top = new FormAttachment(wSuppressErrorLog, margin);
fdComp.right = new FormAttachment(100, -margin);
fdComp.bottom = new FormAttachment(wOk, -margin);
// Limit viewport size so the dialog opens at a reasonable size; content
scrolls inside.
@@ -1056,6 +1071,7 @@ public class ValidatorDialog extends BaseTransformDialog {
wConcatErrors.setSelection(input.isConcatenatingErrors());
wConcatSeparator.setText(Const.NVL(input.getConcatenationSeparator(), ""));
wSuppressLogFailedData.setSelection(input.isSuppressingLogFailedData());
+ wSuppressErrorLog.setSelection(input.isSuppressingErrorLog());
// Select the first available field...
//
@@ -1098,6 +1114,7 @@ public class ValidatorDialog extends BaseTransformDialog {
input.setConcatenatingErrors(wConcatErrors.getSelection());
input.setConcatenationSeparator(wConcatSeparator.getText());
input.setSuppressingLogFailedData(wSuppressLogFailedData.getSelection());
+ input.setSuppressingErrorLog(wSuppressErrorLog.getSelection());
input.setValidations(selectionList);
diff --git
a/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorMeta.java
b/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorMeta.java
index c8ccb6407a..73b7bc5af7 100644
---
a/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorMeta.java
+++
b/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorMeta.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hop.core.CheckResult;
import org.apache.hop.core.Const;
import org.apache.hop.core.ICheckResult;
@@ -91,6 +92,16 @@ public class ValidatorMeta extends
BaseTransformMeta<Validator, ValidatorData> {
injectionKeyDescription = "Validator.Injection.SUPPRESS_LOG_FAILED_DATA")
private boolean suppressingLogFailedData;
+ /**
+ * When true, no log line is written for every rejected row. Error rows are
still sent to the
+ * error handling hop. Defaults to false so existing pipelines keep logging
every rejected row.
+ */
+ @HopMetadataProperty(
+ key = "suppress_error_log",
+ injectionKey = "SUPPRESS_ERROR_LOG",
+ injectionKeyDescription = "Validator.Injection.SUPPRESS_ERROR_LOG")
+ private boolean suppressingErrorLog;
+
/** The standard new validation stream */
@Getter @Setter
private static IStream newValidation =
@@ -112,6 +123,7 @@ public class ValidatorMeta extends
BaseTransformMeta<Validator, ValidatorData> {
this.concatenatingErrors = m.concatenatingErrors;
this.concatenationSeparator = m.concatenationSeparator;
this.suppressingLogFailedData = m.suppressingLogFailedData;
+ this.suppressingErrorLog = m.suppressingErrorLog;
}
@Override
@@ -164,6 +176,44 @@ public class ValidatorMeta extends
BaseTransformMeta<Validator, ValidatorData> {
transformMeta);
remarks.add(cr);
}
+
+ checkInfoTransformsReachEveryCopy(remarks, pipelineMeta, transformMeta,
variables);
+ }
+
+ /**
+ * Every copy of this transform builds its own list of allowed values, so
the transforms
+ * delivering those values have to copy their rows to all the copies instead
of distributing them.
+ * Report that at design time as well, the transform refuses to start
otherwise.
+ */
+ private void checkInfoTransformsReachEveryCopy(
+ List remarks, PipelineMeta pipelineMeta, TransformMeta transformMeta,
IVariables variables) {
+ if (pipelineMeta == null
+ || transformMeta.isPartitioned()
+ || transformMeta.getCopies(variables) <= 1) {
+ return;
+ }
+
+ for (Validation validation : validations) {
+ if (!validation.isSourcingValues()
+ || StringUtils.isEmpty(validation.getSourcingTransformName())) {
+ continue;
+ }
+ TransformMeta sourceTransform =
+ pipelineMeta.findTransform(validation.getSourcingTransformName());
+ if (sourceTransform != null && sourceTransform.isDistributes()) {
+ remarks.add(
+ new CheckResult(
+ ICheckResult.TYPE_RESULT_ERROR,
+ BaseMessages.getString(
+ PKG,
+ "Validator.Exception.InfoTransformIsDistributing",
+ sourceTransform.getName(),
+ transformMeta.getName(),
+ Integer.toString(transformMeta.getCopies(variables)),
+ Const.NVL(validation.getName(),
validation.getFieldName())),
+ transformMeta));
+ }
+ }
}
@Override
diff --git
a/plugins/transforms/validator/src/main/resources/org/apache/hop/pipeline/transforms/validator/messages/messages_en_US.properties
b/plugins/transforms/validator/src/main/resources/org/apache/hop/pipeline/transforms/validator/messages/messages_en_US.properties
index f81ecbec9a..3d873d8f7d 100644
---
a/plugins/transforms/validator/src/main/resources/org/apache/hop/pipeline/transforms/validator/messages/messages_en_US.properties
+++
b/plugins/transforms/validator/src/main/resources/org/apache/hop/pipeline/transforms/validator/messages/messages_en_US.properties
@@ -23,6 +23,7 @@ Validator.Exception.DoesNotEndWithString=During validation of
field ''{0}'' we f
Validator.Exception.DoesNotStartWithString=During validation of field ''{0}''
we found that string value [{1}] is not starting with string [{2}]
Validator.Exception.EndsWithString=During validation of field ''{0}'' we found
that string value [{1}] is ending with string [{2}]
Validator.Exception.HigherThanMaximumValue=During validation of field ''{0}''
we found that value [{1}] is higher than the specified maximum [{2}]
+Validator.Exception.InfoTransformIsDistributing=Transform [{0}] delivers the
allowed values for validation [{3}] but distributes its rows over the {2}
copies of transform [{1}]. Every copy needs all the allowed values, so this
would make valid rows fail. Either run [{1}] in a single copy, or set the data
movement of [{0}] to "Copy data to all transforms".
Validator.Exception.LongerThanMaximumLength=During validation of field ''{0}''
we found that value [{1}] has a longer ({2}) string length than allowed ({3}).
Validator.Exception.LowerThanMinimumValue=During validation of field ''{0}''
we found that value [{1}] is lower than the specified minimum [{2}]
Validator.Exception.MatchingRegExpExpected=During validation of field ''{0}''
we found that value [{1}] is not matching regular expression [{2}]
@@ -39,6 +40,7 @@ Validator.Exception.UnexpectedDataType=During validation of
field ''{0}'' we fou
Validator.Injection.CONCATENATE_ERRORS=This option causes all errors to be
combined into a single output row.
Validator.Injection.CONCATENATION_SEPARATOR=Specify the error separator when
the errors are in one output row.
Validator.Injection.SUPPRESS_LOG_FAILED_DATA=When enabled, failed field/row
values are omitted from validation error messages written to the log. Error
rows are still sent to the error handling hop.
+Validator.Injection.SUPPRESS_ERROR_LOG=When enabled, no log line is written
for every rejected row. Error rows are still sent to the error handling hop.
Validator.Injection.CONVERSION_MASK=Specify the mask to use to convert the
data specified in this validation rule.
Validator.Injection.DATA_TYPE=Specify the data type to verify.
Validator.Injection.DATA_TYPE_VERIFIED=This option causes the specified data
type to be verified.
@@ -73,6 +75,8 @@ ValidatorDialog.ClearButton.Label=Remove validation
ValidatorDialog.ConcatErrors.Label=Output one row, concatenate errors with
separator
ValidatorDialog.SuppressLogFailedData.Label=Do not log failed validation data
ValidatorDialog.SuppressLogFailedData.Tooltip=When enabled, failed field/row
values are omitted from validation error messages (shown as "?"). Error
messages are still logged. Error rows are still sent to the error handling hop
when configured.
+ValidatorDialog.SuppressErrorLog.Label=Do not log a line for every rejected row
+ValidatorDialog.SuppressErrorLog.Tooltip=When enabled, the transform no longer
writes an error line to the log for every row it rejects. Error rows are still
sent to the error handling hop when configured. On streams where many rows are
rejected this log line costs far more than the validation itself.
ValidatorDialog.ConversionMask.Label=Conversion mask
ValidatorDialog.DataGroup.Label=Data
ValidatorDialog.DataType.Label=Data type
diff --git
a/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidatorTest.java
b/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidatorTest.java
index 8d7b8b203c..c862ed9289 100644
---
a/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidatorTest.java
+++
b/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidatorTest.java
@@ -30,6 +30,7 @@ import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
@@ -47,6 +48,11 @@ import org.apache.hop.core.row.value.ValueMetaInteger;
import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.partition.PartitionSchema;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transform.TransformMetaDataCombi;
+import org.apache.hop.pipeline.transform.TransformPartitioningMeta;
import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
@@ -814,6 +820,158 @@ class ValidatorTest {
verify(validator).putRow(any(IRowMeta.class), eq(new Object[] {"ABZ"}));
}
+ @Test
+ void testInitFailsWhenInfoTransformDistributesOverMultipleCopies() throws
Exception {
+ ValidatorMeta meta = createMeta(sourcingRule("allowed values",
"allowed_value"));
+ Validator validator = createValidatorWithInfoTransform(meta, true, 4,
false);
+
+ assertFalse(validator.init());
+ }
+
+ @Test
+ void testInitSucceedsWhenInfoTransformCopiesToAllCopies() throws Exception {
+ ValidatorMeta meta = createMeta(sourcingRule("allowed values",
"allowed_value"));
+ Validator validator = createValidatorWithInfoTransform(meta, false, 4,
false);
+
+ assertTrue(validator.init());
+ }
+
+ @Test
+ void testInitSucceedsWhenRunningInASingleCopy() throws Exception {
+ ValidatorMeta meta = createMeta(sourcingRule("allowed values",
"allowed_value"));
+ Validator validator = createValidatorWithInfoTransform(meta, true, 1,
false);
+
+ assertTrue(validator.init());
+ }
+
+ @Test
+ void testInitSucceedsForPartitionedTransforms() throws Exception {
+ ValidatorMeta meta = createMeta(sourcingRule("allowed values",
"allowed_value"));
+ Validator validator = createValidatorWithInfoTransform(meta, true, 4,
true);
+
+ assertTrue(validator.init());
+ }
+
+ @Test
+ void testInitIgnoresDistributionWhenValuesAreNotSourced() throws Exception {
+ Validation rule = new Validation();
+ rule.setName("list");
+ rule.setFieldName("value");
+ rule.setNullAllowed(true);
+ rule.setDataType("String");
+ rule.setAllowedValues(List.of("A", "B"));
+
+ ValidatorMeta meta = createMeta(rule);
+ Validator validator = createValidatorWithInfoTransform(meta, true, 4,
false);
+
+ assertTrue(validator.init());
+ }
+
+ @Test
+ void testCaseInsensitiveComparisonStillMatchesAllowedValues() throws
Exception {
+ Validation rule = new Validation();
+ rule.setName("list");
+ rule.setFieldName("value");
+ rule.setNullAllowed(true);
+ rule.setDataType("String");
+ rule.setAllowedValues(List.of("A", "B"));
+
+ ValidatorMeta meta = createMeta(rule);
+
+ // A case insensitive field can't use the hash based lookup, the linear
comparison has to kick
+ // in
+ ValueMetaString valueMeta = new ValueMetaString("value");
+ valueMeta.setCaseInsensitive(true);
+ RowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(valueMeta);
+
+ Validator validator = createInitializedValidator(meta, rowMeta, true);
+ doReturn(new Object[] {"a"}).doReturn(null).when(validator).getRow();
+ doNothing().when(validator).putRow(any(IRowMeta.class),
any(Object[].class));
+
+ assertTrue(validator.processRow());
+ verify(validator).putRow(any(IRowMeta.class), eq(new Object[] {"a"}));
+ verify(validator, never())
+ .putError(
+ any(IRowMeta.class),
+ any(Object[].class),
+ anyLong(),
+ anyString(),
+ anyString(),
+ anyString());
+ }
+
+ @Test
+ void testErrorDescriptionsAreLoggedByDefault() throws Exception {
+ ValidatorMeta meta = createMeta(nullNotAllowed());
+ Validator validator = createInitializedValidator(meta, stringRowMeta(),
true);
+
+ assertTrue(validator.isLoggingErrorDescriptions());
+ }
+
+ @Test
+ void testSuppressErrorLogStopsTheLogLinePerRejectedRow() throws Exception {
+ ValidatorMeta meta = createMeta(nullNotAllowed());
+ meta.setSuppressingErrorLog(true);
+ Validator validator = createInitializedValidator(meta, stringRowMeta(),
true);
+
+ assertFalse(validator.isLoggingErrorDescriptions());
+
+ // the error row itself still has to reach the error handling hop
+ doReturn(new Object[] {null}).doReturn(null).when(validator).getRow();
+ stubPutError(validator);
+
+ assertTrue(validator.processRow());
+ verify(validator)
+ .putError(
+ any(IRowMeta.class),
+ any(Object[].class),
+ eq(1L),
+ anyString(),
+ eq("value"),
+ eq("KVD001"));
+ }
+
+ private Validator createValidatorWithInfoTransform(
+ ValidatorMeta meta, boolean infoTransformDistributes, int copies,
boolean partitioned)
+ throws Exception {
+ TransformMeta infoTransform = new TransformMeta("allowed values", new
ValidatorMeta());
+ infoTransform.setDistributes(infoTransformDistributes);
+
+ TransformMetaDataCombi combi = new TransformMetaDataCombi();
+ combi.transformMeta = infoTransform;
+ combi.transformName = infoTransform.getName();
+ doReturn(List.of(combi)).when((LocalPipelineEngine)
helper.pipeline).getTransforms();
+
+
when(helper.transformMeta.getCopies(any(IVariables.class))).thenReturn(copies);
+ when(helper.transformMeta.isPartitioned()).thenReturn(partitioned);
+ if (partitioned) {
+ PartitionSchema partitionSchema = mock(PartitionSchema.class);
+ when(partitionSchema.calculatePartitionIds(any(IVariables.class)))
+ .thenReturn(new ArrayList<>(List.of("p0", "p1", "p2", "p3")));
+ TransformPartitioningMeta partitioningMeta =
mock(TransformPartitioningMeta.class);
+ when(partitioningMeta.getPartitionSchema()).thenReturn(partitionSchema);
+
when(helper.transformMeta.getTransformPartitioningMeta()).thenReturn(partitioningMeta);
+ }
+ when(helper.pipelineMeta.getTransformFields(any(IVariables.class),
anyString()))
+ .thenReturn(stringRowMeta());
+
+ return new Validator(
+ helper.transformMeta, meta, new ValidatorData(), 0,
helper.pipelineMeta, helper.pipeline);
+ }
+
+ private static Validation sourcingRule(String sourcingTransformName, String
sourcingField) {
+ Validation rule = new Validation();
+ rule.setName("sourced list");
+ rule.setFieldName("value");
+ rule.setNullAllowed(true);
+ rule.setDataType("String");
+ rule.setSourcingValues(true);
+ rule.setSourcingTransformName(sourcingTransformName);
+ rule.setSourcingField(sourcingField);
+ return rule;
+ }
+
private Validator createInitializedValidator(
ValidatorMeta meta, RowMeta rowMeta, boolean errorHandling) throws
Exception {
when(helper.transformMeta.isDoingErrorHandling()).thenReturn(errorHandling);
