This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/master by this push:
new 798111d3e2 HOP-4074 Value mapper: support the usage of variables
new d838855210 Merge pull request #1661 from nadment/HOP-4074B
798111d3e2 is described below
commit 798111d3e23ffe809315c753162b09c142179585
Author: Nicolas Adment <[email protected]>
AuthorDate: Thu Aug 25 23:22:43 2022 +0200
HOP-4074 Value mapper: support the usage of variables
---
.../pages/pipeline/transforms/valuemapper.adoc | 4 +-
integration-tests/transforms/0045-value-mapper.hpl | 217 +++++++++++++++++++++
integration-tests/transforms/0045-value-mapper.hwf | 79 ++++++++
.../transforms/datasets/golden-value-mapper.csv | 9 +
.../metadata/dataset/golden-value-mapper.json | 40 ++++
.../metadata/unit-test/0045-value-mapper UNIT.json | 43 ++++
.../transforms/valuemapper/ValueMapper.java | 25 ++-
.../transforms/valuemapper/ValueMapperData.java | 11 +-
.../transforms/valuemapper/ValueMapperDialog.java | 8 +-
9 files changed, 413 insertions(+), 23 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/valuemapper.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/valuemapper.adoc
index 69b6a1fdb6..009098fa66 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/valuemapper.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/valuemapper.adoc
@@ -62,9 +62,11 @@ The following properties are used to define the mappings:
|Fieldname to use|Field to use as the mapping source
|Target field name|Field to use as the mapping Target
|Default upon non-matching|Defines a default value for situations where the
source value is not empty, but there is no match
-|Field values table|Contains the mapping of source value to converted target
value
+|Field values table|Contains the mapping of source value to converted target
value.
|===
+TIP: If there is an empty source value defined, we map null or empty string to
the target value. Only one empty mapping is allowed.
+
== Metadata Injection Support
All fields of this transform support metadata injection.
diff --git a/integration-tests/transforms/0045-value-mapper.hpl
b/integration-tests/transforms/0045-value-mapper.hpl
new file mode 100644
index 0000000000..67639a340e
--- /dev/null
+++ b/integration-tests/transforms/0045-value-mapper.hpl
@@ -0,0 +1,217 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0045-value-mapper</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters>
+ <parameter>
+ <name>NOT_DEFINED</name>
+ <default_value>Not defined</default_value>
+ <description/>
+ </parameter>
+ <parameter>
+ <name>NOT_FOUND</name>
+ <default_value>Not found</default_value>
+ <description/>
+ </parameter>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2022/06/30 15:35:29.264</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2022/06/30 15:35:29.264</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Data grid</from>
+ <to>Value mapper</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Value mapper</from>
+ <to>Dummy</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Data grid</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>N</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>3</length>
+ <name>IATA_Code</name>
+ <precision>-1</precision>
+ <type>String</type>
+ </field>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>50</length>
+ <name>Airport_Name</name>
+ <precision>-1</precision>
+ <type>String</type>
+ </field>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>2</length>
+ <name>Country_Code</name>
+ <precision>-1</precision>
+ <type>String</type>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>JFK</item>
+ <item>John F. Kennedy International Airport</item>
+ <item>US</item>
+ </line>
+ <line>
+ <item>LAX</item>
+ <item>Los Angeles International Airport</item>
+ <item/>
+ </line>
+ <line>
+ <item>ORD</item>
+ <item>O'Hare International Airport</item>
+ <item>US</item>
+ </line>
+ <line>
+ <item>YYZ</item>
+ <item>Lester B. Pearson International Airport</item>
+ <item>CA</item>
+ </line>
+ <line>
+ <item>LHR</item>
+ <item>Heathrow Airport</item>
+ <item>UK</item>
+ </line>
+ <line>
+ <item>BRU</item>
+ <item>Brussels Airport</item>
+ <item>BE</item>
+ </line>
+ <line>
+ <item>CDG</item>
+ <item>Paris Charles de Gaulle Airport</item>
+ <item>FR</item>
+ </line>
+ <line>
+ <item>HRG</item>
+ <item>Hurghada</item>
+ <item>EG</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Dummy</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>368</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Value mapper</name>
+ <type>ValueMapper</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <field_to_use>Country_Code</field_to_use>
+ <target_field>Country_Name</target_field>
+ <non_match_default>[${NOT_FOUND}]</non_match_default>
+ <fields>
+ <field>
+ <source_value/>
+ <target_value>[${NOT_DEFINED}]</target_value>
+ </field>
+ <field>
+ <source_value>BE</source_value>
+ <target_value>Belgium</target_value>
+ </field>
+ <field>
+ <source_value>CA</source_value>
+ <target_value>Canada</target_value>
+ </field>
+ <field>
+ <source_value>FR</source_value>
+ <target_value>France</target_value>
+ </field>
+ <field>
+ <source_value>UK</source_value>
+ <target_value>United Kingdom</target_value>
+ </field>
+ <field>
+ <source_value>US</source_value>
+ <target_value>United States</target_value>
+ </field>
+ <field>
+ <source_value>DE</source_value>
+ <target_value>German</target_value>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>240</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/0045-value-mapper.hwf
b/integration-tests/transforms/0045-value-mapper.hwf
new file mode 100644
index 0000000000..e5995e08ea
--- /dev/null
+++ b/integration-tests/transforms/0045-value-mapper.hwf
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>0045-value-mapper</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2022/04/29 10:42:16.470</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2022/04/29 10:42:16.470</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <xloc>50</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run ValueMapper Unit Test</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0045-value-mapper UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>192</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Run ValueMapper Unit Test</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/transforms/datasets/golden-value-mapper.csv
b/integration-tests/transforms/datasets/golden-value-mapper.csv
new file mode 100644
index 0000000000..23edc538fc
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-value-mapper.csv
@@ -0,0 +1,9 @@
+IATA_Code,Airport_Name,Country_Code,Country_Name
+JFK,John F. Kennedy International Airport,US,United States
+LAX,Los Angeles International Airport,,[Not defined]
+ORD,O'Hare International Airport,US,United States
+YYZ,Lester B. Pearson International Airport,CA,Canada
+LHR,Heathrow Airport,UK,United Kingdom
+BRU,Brussels Airport,BE,Belgium
+CDG,Paris Charles de Gaulle Airport,FR,France
+HRG,Hurghada,EG,[Not found]
diff --git
a/integration-tests/transforms/metadata/dataset/golden-value-mapper.json
b/integration-tests/transforms/metadata/dataset/golden-value-mapper.json
new file mode 100644
index 0000000000..61ee7885e6
--- /dev/null
+++ b/integration-tests/transforms/metadata/dataset/golden-value-mapper.json
@@ -0,0 +1,40 @@
+{
+ "base_filename": "golden-value-mapper.csv",
+ "name": "golden-value-mapper",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": 3,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "IATA_Code"
+ },
+ {
+ "field_comment": "",
+ "field_length": 50,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "Airport_Name"
+ },
+ {
+ "field_comment": "",
+ "field_length": 2,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "Country_Code"
+ },
+ {
+ "field_comment": "",
+ "field_length": 16,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "Country_Name"
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git a/integration-tests/transforms/metadata/unit-test/0045-value-mapper
UNIT.json b/integration-tests/transforms/metadata/unit-test/0045-value-mapper
UNIT.json
new file mode 100644
index 0000000000..52d9306dd7
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0045-value-mapper
UNIT.json
@@ -0,0 +1,43 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": true,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "IATA_Code",
+ "data_set_field": "IATA_Code"
+ },
+ {
+ "transform_field": "Airport_Name",
+ "data_set_field": "Airport_Name"
+ },
+ {
+ "transform_field": "Country_Code",
+ "data_set_field": "Country_Code"
+ },
+ {
+ "transform_field": "Country_Name",
+ "data_set_field": "Country_Name"
+ }
+ ],
+ "field_order": [
+ "IATA_Code",
+ "Airport_Name",
+ "Country_Code",
+ "Country_Name"
+ ],
+ "transform_name": "Dummy",
+ "data_set_name": "golden-value-mapper"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0045-value-mapper UNIT",
+ "description": "",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename": "./0045-value-mapper.hpl",
+ "test_type": "UNIT_TEST"
+}
\ No newline at end of file
diff --git
a/plugins/transforms/valuemapper/src/main/java/org/apache/hop/pipeline/transforms/valuemapper/ValueMapper.java
b/plugins/transforms/valuemapper/src/main/java/org/apache/hop/pipeline/transforms/valuemapper/ValueMapper.java
index 3d9d8ac955..bc7f3393a1 100644
---
a/plugins/transforms/valuemapper/src/main/java/org/apache/hop/pipeline/transforms/valuemapper/ValueMapper.java
+++
b/plugins/transforms/valuemapper/src/main/java/org/apache/hop/pipeline/transforms/valuemapper/ValueMapper.java
@@ -26,8 +26,7 @@ import org.apache.hop.pipeline.Pipeline;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransform;
import org.apache.hop.pipeline.transform.TransformMeta;
-
-import java.util.Hashtable;
+import java.util.HashMap;
/** Convert Values in a certain fields to other values */
public class ValueMapper extends BaseTransform<ValueMapperMeta,
ValueMapperData> {
@@ -84,8 +83,8 @@ public class ValueMapper extends
BaseTransform<ValueMapperMeta, ValueMapperData>
//
for (int i = 0; i < meta.getSourceValue().length; i++) {
if (Utils.isEmpty(meta.getSourceValue()[i])) {
- if (data.emptyFieldIndex < 0) {
- data.emptyFieldIndex = i;
+ if (data.emptyFieldValue==null) {
+ data.emptyFieldValue = resolve(meta.getTargetValue()[i]);
} else {
throw new HopException(
BaseMessages.getString(
@@ -110,14 +109,14 @@ public class ValueMapper extends
BaseTransform<ValueMapperMeta, ValueMapperData>
// Null/Empty mapping to value...
//
- if (data.emptyFieldIndex >= 0 && (r[data.keynr] == null ||
Utils.isEmpty(source))) {
- target = meta.getTargetValue()[data.emptyFieldIndex]; // that's all
there is to it.
+ if (data.emptyFieldValue!=null && (r[data.keynr] == null ||
Utils.isEmpty(source))) {
+ target = data.emptyFieldValue; // that's all there is to it.
} else {
if (!Utils.isEmpty(source)) {
- target = data.hashtable.get(source);
+ target = data.mapValues.get(source);
if (nonMatchActivated && target == null) {
// If we do non matching and we don't have a match
- target = meta.getNonMatchDefault();
+ target = data.nonMatchDefault;
}
}
}
@@ -172,24 +171,24 @@ public class ValueMapper extends
BaseTransform<ValueMapperMeta, ValueMapperData>
public boolean init() {
if (super.init()) {
- data.hashtable = new Hashtable<>();
- data.emptyFieldIndex = -1;
+ data.mapValues = new HashMap<>();
if (!Utils.isEmpty(meta.getNonMatchDefault())) {
nonMatchActivated = true;
+ data.nonMatchDefault = resolve(meta.getNonMatchDefault());
}
// Add all source to target mappings in here...
for (int i = 0; i < meta.getSourceValue().length; i++) {
String src = meta.getSourceValue()[i];
- String tgt = meta.getTargetValue()[i];
+ String tgt = this.resolve(meta.getTargetValue()[i]);
if (!Utils.isEmpty(src) && !Utils.isEmpty(tgt)) {
- data.hashtable.put(src, tgt);
+ data.mapValues.put(src, tgt);
} else {
if (Utils.isEmpty(tgt)) {
// allow target to be set to null since 3.0
- data.hashtable.put(src, "");
+ data.mapValues.put(src, "");
}
}
}
diff --git
a/plugins/transforms/valuemapper/src/main/java/org/apache/hop/pipeline/transforms/valuemapper/ValueMapperData.java
b/plugins/transforms/valuemapper/src/main/java/org/apache/hop/pipeline/transforms/valuemapper/ValueMapperData.java
index dc4b0771fe..dc67a72858 100644
---
a/plugins/transforms/valuemapper/src/main/java/org/apache/hop/pipeline/transforms/valuemapper/ValueMapperData.java
+++
b/plugins/transforms/valuemapper/src/main/java/org/apache/hop/pipeline/transforms/valuemapper/ValueMapperData.java
@@ -22,8 +22,7 @@ import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.pipeline.transform.BaseTransformData;
import org.apache.hop.pipeline.transform.ITransformData;
-
-import java.util.Hashtable;
+import java.util.HashMap;
public class ValueMapperData extends BaseTransformData implements
ITransformData {
public IRowMeta previousMeta;
@@ -31,9 +30,9 @@ public class ValueMapperData extends BaseTransformData
implements ITransformData
public int keynr;
- public Hashtable<String, String> hashtable;
-
- public int emptyFieldIndex;
+ public HashMap<String, String> mapValues;
+ public String nonMatchDefault;
+ public String emptyFieldValue;
public IValueMeta stringMeta;
public IValueMeta outputValueMeta;
@@ -42,7 +41,7 @@ public class ValueMapperData extends BaseTransformData
implements ITransformData
public ValueMapperData() {
super();
- hashtable = null;
+ mapValues = null;
stringMeta = new ValueMetaString("string");
}
diff --git
a/plugins/transforms/valuemapper/src/main/java/org/apache/hop/pipeline/transforms/valuemapper/ValueMapperDialog.java
b/plugins/transforms/valuemapper/src/main/java/org/apache/hop/pipeline/transforms/valuemapper/ValueMapperDialog.java
index 30032194d6..29794be3bd 100644
---
a/plugins/transforms/valuemapper/src/main/java/org/apache/hop/pipeline/transforms/valuemapper/ValueMapperDialog.java
+++
b/plugins/transforms/valuemapper/src/main/java/org/apache/hop/pipeline/transforms/valuemapper/ValueMapperDialog.java
@@ -30,6 +30,7 @@ import org.apache.hop.ui.core.dialog.BaseDialog;
import org.apache.hop.ui.core.dialog.ErrorDialog;
import org.apache.hop.ui.core.widget.ColumnInfo;
import org.apache.hop.ui.core.widget.TableView;
+import org.apache.hop.ui.core.widget.TextVar;
import org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.CCombo;
@@ -51,7 +52,7 @@ public class ValueMapperDialog extends BaseTransformDialog
implements ITransform
private Text wTargetFieldName;
- private Text wNonMatchDefault;
+ private TextVar wNonMatchDefault;
private TableView wFields;
@@ -174,7 +175,7 @@ public class ValueMapperDialog extends BaseTransformDialog
implements ITransform
fdlNonMatchDefault.right = new FormAttachment(middle, -margin);
fdlNonMatchDefault.top = new FormAttachment(wTargetFieldName, margin);
wlNonMatchDefault.setLayoutData(fdlNonMatchDefault);
- wNonMatchDefault = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ wNonMatchDefault = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT |
SWT.BORDER);
props.setLook(wNonMatchDefault);
wNonMatchDefault.addModifyListener(lsMod);
FormData fdNonMatchDefault = new FormData();
@@ -205,7 +206,8 @@ public class ValueMapperDialog extends BaseTransformDialog
implements ITransform
BaseMessages.getString(PKG,
"ValueMapperDialog.Fields.Column.TargetValue"),
ColumnInfo.COLUMN_TYPE_TEXT,
false);
-
+ colinf[1].setUsingVariables(true);
+
wFields =
new TableView(
variables,
