This is an automated email from the ASF dual-hosted git repository.
mattcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 9f0219b9c9 harden normalizer with mixed field types, fixes #3636
(#8549)
9f0219b9c9 is described below
commit 9f0219b9c91c117489674e8f52540b67e9e1e6b4
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Wed Sep 23 17:12:18 2026 +0200
harden normalizer with mixed field types, fixes #3636 (#8549)
---
.../pages/pipeline/transforms/rownormaliser.adoc | 44 +-
.../transforms/0020-row-normaliser-mixed-types.hpl | 249 +++++++++++
.../datasets/golden-row-normaliser-mixed-types.csv | 5 +
.../transforms/main-0020-row-normaliser.hwf | 3 +
.../dataset/golden-row-normaliser-mixed-types.json | 40 ++
.../0020-row-normaliser-mixed-types UNIT.json | 43 ++
.../pipeline/transforms/normaliser/Normaliser.java | 59 ++-
.../transforms/normaliser/NormaliserData.java | 14 +-
.../transforms/normaliser/NormaliserMeta.java | 149 ++++++-
.../normaliser/messages/messages_en_US.properties | 2 +
.../transforms/normaliser/NormaliserMetaTest.java | 183 ++++++++
.../transforms/normaliser/NormaliserTest.java | 487 +++++++++++++++------
12 files changed, 1094 insertions(+), 184 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rownormaliser.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rownormaliser.adoc
index e4b75cc0b1..8c92709504 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rownormaliser.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rownormaliser.adoc
@@ -27,7 +27,20 @@ The Row Normaliser transform converts the columns of an
input stream into rows.
You can use this transform to normalize repeating groups of columns.
-*Important*: When combining multiple columns with different meta types (e.g.,
String and Integer) into a new field, no automatic type conversion is
performed. Instead the first meta type is set. This lack of conversion may lead
to issues with subsequent transformations on the resulting data rows. It is
strongly advised to ensure that the data types of values being put into the
same field are aligned before normalization.
+For every row it reads, the transform writes one row per type. Each input
field goes into the new field it names on the row of its type, whatever order
the fields are listed in. A type that has no field for one of the new fields
leaves that field empty (`null`) on its rows.
+
+A type can fill each new field from only one input field: listing two fields
with the same type and the same new field is an error.
+
+== Data types of the new fields
+
+A new field is filled from a different input field on every row, so its data
type depends on all of them:
+
+* When the input fields filling a new field share a data type, the new field
has that type, with the length, precision and format of the first of them. The
values are passed on unchanged.
+* When they do not, the new field is a String, and every value is converted to
text using the format of the input field it comes from. To control how a number
or a date is written, set the format on that input field before this transform,
for example with a xref:pipeline/transforms/selectvalues.adoc[Select Values]
transform.
+
+For example, normalising a String, an Integer and a Number into one field
gives a String field holding `a`, `1` and `1.2`. **Verify** on this transform
warns about every new field that is filled from input fields of different data
types.
+
+NOTE: Before Hop 2.20, a new field took the data type of the first input field
filling it, and the values of the other input fields were passed on unchanged.
Rows with a value that did not match that type failed further down the
pipeline, for example in a Sort rows transform that writes rows to disk.
== Options
@@ -93,3 +106,32 @@ Similar to example 1, but remove the **RecordID** field
from the **Fields table*
|824-21-0000|LastName|Ledner
|824-21-0000|City|Scottieview
|===
+
+=== Normalized data (example 3)
+Several new fields can be filled at once. With this input:
+
+[options="header"]
+|===
+|Date|PR1_SL|PR1_NR|PR2_SL|PR2_NR
+|2003-01-01|100|5|250|10
+|===
+
+set **Typefield** = "Product" and fill the **Fields table** like this:
+
+[options="header"]
+|===
+|Fieldname|Type|New field
+|PR1_SL|Product1|Sales
+|PR1_NR|Product1|Number
+|PR2_NR|Product2|Number
+|PR2_SL|Product2|Sales
+|===
+
+The result has one row per product, each value in the new field it names:
+
+[options="header"]
+|===
+|Date|Product|Sales|Number
+|2003-01-01|Product1|100|5
+|2003-01-01|Product2|250|10
+|===
diff --git a/integration-tests/transforms/0020-row-normaliser-mixed-types.hpl
b/integration-tests/transforms/0020-row-normaliser-mixed-types.hpl
new file mode 100644
index 0000000000..6507e8b556
--- /dev/null
+++ b/integration-tests/transforms/0020-row-normaliser-mixed-types.hpl
@@ -0,0 +1,249 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0020-row-normaliser-mixed-types</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Issue #3636: normalised fields filled from input fields of
different types, listed in a different order for each type. Each normalised
field is a String holding the text of every value, and every value lands in the
field it names. The sort spills every row to disk, which is where a value of
the wrong type used to fail.</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/23 11:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/23 11:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Sample data</from>
+ <to>Normalise mixed types</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Normalise mixed types</from>
+ <to>Sort through disk</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Sort through disk</from>
+ <to>Verify</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Sample data</name>
+ <type>DataGrid</type>
+ <description>The price names its format and decimal symbol, so its text
does not depend on the platform locale.</description>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>1</item>
+ <item>alpha</item>
+ <item>10</item>
+ <item>A1</item>
+ <item>2.50</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>beta</item>
+ <item>20</item>
+ <item>B2</item>
+ <item>3.75</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <format></format>
+ <currency/>
+ <decimal></decimal>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>text</name>
+ <type>String</type>
+ <format></format>
+ <currency/>
+ <decimal></decimal>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>amount</name>
+ <type>Integer</type>
+ <format></format>
+ <currency/>
+ <decimal></decimal>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>code</name>
+ <type>String</type>
+ <format></format>
+ <currency/>
+ <decimal></decimal>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>price</name>
+ <type>Number</type>
+ <format>0.00</format>
+ <currency/>
+ <decimal>.</decimal>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Normalise mixed types</name>
+ <type>Normaliser</type>
+ <description>X is filled from a String and a Number, Y from an Integer and
a String. Type second lists Y before X.</description>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>text</name>
+ <value>first</value>
+ <norm>X</norm>
+ </field>
+ <field>
+ <name>amount</name>
+ <value>first</value>
+ <norm>Y</norm>
+ </field>
+ <field>
+ <name>code</name>
+ <value>second</value>
+ <norm>Y</norm>
+ </field>
+ <field>
+ <name>price</name>
+ <value>second</value>
+ <norm>X</norm>
+ </field>
+ </fields>
+ <typefield>typefield</typefield>
+ <attributes/>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Sort through disk</name>
+ <type>SortRows</type>
+ <description>Sorts one row in memory at a time, so every row is written to
a temporary file and read back.</description>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <directory>${java.io.tmpdir}</directory>
+ <prefix>out</prefix>
+ <sort_size>1</sort_size>
+ <free_memory/>
+ <compress>N</compress>
+ <compress_variable/>
+ <unique_rows>N</unique_rows>
+ <fields>
+ <field>
+ <name>id</name>
+ <ascending>Y</ascending>
+ <case_sensitive>N</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ <field>
+ <name>typefield</name>
+ <ascending>Y</ascending>
+ <case_sensitive>N</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>416</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Verify</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>576</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-row-normaliser-mixed-types.csv
b/integration-tests/transforms/datasets/golden-row-normaliser-mixed-types.csv
new file mode 100644
index 0000000000..ce5afc8561
--- /dev/null
+++
b/integration-tests/transforms/datasets/golden-row-normaliser-mixed-types.csv
@@ -0,0 +1,5 @@
+id,typefield,X,Y
+1,first,alpha,10
+1,second,2.50,A1
+2,first,beta,20
+2,second,3.75,B2
diff --git a/integration-tests/transforms/main-0020-row-normaliser.hwf
b/integration-tests/transforms/main-0020-row-normaliser.hwf
index 5ba961bcbf..bc6d6a3766 100644
--- a/integration-tests/transforms/main-0020-row-normaliser.hwf
+++ b/integration-tests/transforms/main-0020-row-normaliser.hwf
@@ -60,6 +60,9 @@ limitations under the License.
<test_name>
<name>0020-row-normaliser-multiple-targets UNIT</name>
</test_name>
+ <test_name>
+ <name>0020-row-normaliser-mixed-types UNIT</name>
+ </test_name>
</test_names>
<parallel>N</parallel>
<xloc>272</xloc>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-row-normaliser-mixed-types.json
b/integration-tests/transforms/metadata/dataset/golden-row-normaliser-mixed-types.json
new file mode 100644
index 0000000000..0cb73c0f6b
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-row-normaliser-mixed-types.json
@@ -0,0 +1,40 @@
+{
+ "base_filename": "golden-row-normaliser-mixed-types.csv",
+ "name": "golden-row-normaliser-mixed-types",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_format": "####0;-####0",
+ "field_name": "id"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "typefield"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "X"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "Y"
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/unit-test/0020-row-normaliser-mixed-types
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0020-row-normaliser-mixed-types
UNIT.json
new file mode 100644
index 0000000000..eafdb7e99a
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0020-row-normaliser-mixed-types
UNIT.json
@@ -0,0 +1,43 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": true,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "typefield",
+ "data_set_field": "typefield"
+ },
+ {
+ "transform_field": "X",
+ "data_set_field": "X"
+ },
+ {
+ "transform_field": "Y",
+ "data_set_field": "Y"
+ }
+ ],
+ "field_order": [
+ "id",
+ "typefield",
+ "X",
+ "Y"
+ ],
+ "transform_name": "Verify",
+ "data_set_name": "golden-row-normaliser-mixed-types"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0020-row-normaliser-mixed-types UNIT",
+ "description": "",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename": "./0020-row-normaliser-mixed-types.hpl",
+ "test_type": "UNIT_TEST"
+}
\ No newline at end of file
diff --git
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/Normaliser.java
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/Normaliser.java
index 9f6919ec94..dfda2c2768 100644
---
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/Normaliser.java
+++
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/Normaliser.java
@@ -53,8 +53,6 @@ public class Normaliser extends BaseTransform<NormaliserMeta,
NormaliserData> {
return false;
}
- List<Integer> normFieldList;
-
if (first) { // INITIALISE
first = false;
@@ -62,10 +60,20 @@ public class Normaliser extends
BaseTransform<NormaliserMeta, NormaliserData> {
data.inputRowMeta = getInputRowMeta();
data.outputRowMeta = data.inputRowMeta.clone();
meta.getFields(data.outputRowMeta, getTransformName(), null, null, this,
metadataProvider);
- data.typeToFieldIndex = new HashMap<>();
+ data.typeToPlacements = new HashMap<>();
String typeValue;
int dataFieldNr;
+ // The normalised fields are the last ones of the output row, in this
order.
+ //
+ List<String> normNames = meta.getNormalisedFieldNames();
+ int firstNormIndex = data.outputRowMeta.size() - normNames.size();
+
+ String duplicate = meta.getDuplicateMapping();
+ if (duplicate != null) {
+ throw new HopException(duplicate);
+ }
+
// Get a unique list of occurrences...
//
data.type_occ = new ArrayList<>();
@@ -80,15 +88,9 @@ public class Normaliser extends
BaseTransform<NormaliserMeta, NormaliserData> {
data.maxlen = typeValue.length();
}
- // This next section creates a map of arraylist objects. The key is
the Type in the
- // Normaliser
- // and the ArrayList is the list of indexes on the row of all fields
that get normalized
- // under that Type.
- // This eliminates the inner loop that iterated over all the fields
finding the fields
- // associated with the Type.
- // On a test data set with 2500 fields and about 36000 input rows
(outputting over 22m
- // rows), the time went from
- // 12min to about 1min 35sec.
+ // For every type, the fields that get normalised under it and where
each one goes: into
+ // the normalised field it names, whatever order the fields are listed
in.
+ //
dataFieldNr = data.inputRowMeta.indexOfValue(field.getName());
if (dataFieldNr < 0) {
logError(
@@ -98,12 +100,17 @@ public class Normaliser extends
BaseTransform<NormaliserMeta, NormaliserData> {
stopAll();
return false;
}
- normFieldList = data.typeToFieldIndex.get(typeValue);
- if (normFieldList == null) {
- normFieldList = new ArrayList<>();
- data.typeToFieldIndex.put(typeValue, normFieldList);
- }
- normFieldList.add(dataFieldNr);
+ int outputIndex = firstNormIndex + normNames.indexOf(field.getNorm());
+ IValueMeta source = data.inputRowMeta.getValueMeta(dataFieldNr);
+ IValueMeta target = data.outputRowMeta.getValueMeta(outputIndex);
+ boolean copy =
+ source.getType() == target.getType()
+ && source.getStorageType() == target.getStorageType();
+ data.typeToPlacements
+ .computeIfAbsent(typeValue, k -> new ArrayList<>())
+ .add(
+ new NormaliserData.Placement(
+ dataFieldNr, outputIndex, source, copy ? null : target));
}
// Which fields are not impacted? We can just copy these, leave them
alone.
@@ -159,11 +166,17 @@ public class Normaliser extends
BaseTransform<NormaliserMeta, NormaliserData> {
// Then add the normalized fields...
//
- normFieldList = data.typeToFieldIndex.get(typeValue);
- int normFieldListSz = normFieldList.size();
- for (Integer integer : normFieldList) {
- value = r[integer];
- outputRowData[outputIndex++] = value;
+ for (NormaliserData.Placement placement :
data.typeToPlacements.get(typeValue)) {
+ value = r[placement.inputIndex()];
+ IValueMeta target = placement.target();
+ if (target != null) {
+ // Either the same type in another storage, or text for a field of
mixed types.
+ value =
+ target.getType() == placement.source().getType()
+ ? placement.source().convertToNormalStorageType(value)
+ : target.convertData(placement.source(), value);
+ }
+ outputRowData[placement.outputIndex()] = value;
}
// The row is constructed, now give it to the next transform(s)...
diff --git
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserData.java
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserData.java
index 9c28694f9b..6bb2c777d8 100644
---
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserData.java
+++
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserData.java
@@ -20,6 +20,7 @@ package org.apache.hop.pipeline.transforms.normaliser;
import java.util.List;
import java.util.Map;
import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.pipeline.transform.BaseTransformData;
import org.apache.hop.pipeline.transform.ITransformData;
@@ -28,7 +29,7 @@ public class NormaliserData extends BaseTransformData
implements ITransformData
public List<String> type_occ;
public int maxlen;
public List<Integer> copyFieldnrs;
- Map<String, List<Integer>> typeToFieldIndex;
+ Map<String, List<Placement>> typeToPlacements;
public IRowMeta inputRowMeta;
public IRowMeta outputRowMeta;
@@ -38,4 +39,15 @@ public class NormaliserData extends BaseTransformData
implements ITransformData
type_occ = null;
}
+
+ /**
+ * Where one input field goes on the output row of its type.
+ *
+ * @param inputIndex the input field
+ * @param outputIndex the normalised field it fills
+ * @param source the input field's metadata
+ * @param target the normalised field's metadata when the value has to be
converted into it, null
+ * when it is copied as it is
+ */
+ record Placement(int inputIndex, int outputIndex, IValueMeta source,
IValueMeta target) {}
}
diff --git
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMeta.java
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMeta.java
index 23dd5ac722..42469332e6 100644
---
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMeta.java
+++
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMeta.java
@@ -20,6 +20,7 @@ package org.apache.hop.pipeline.transforms.normaliser;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
import lombok.Getter;
import lombok.Setter;
@@ -112,6 +113,67 @@ public class NormaliserMeta extends
BaseTransformMeta<Normaliser, NormaliserData
this.normaliserFields = new ArrayList<>();
}
+ /** The names of the normalised fields, in output order: in the order they
first appear. */
+ public List<String> getNormalisedFieldNames() {
+ List<String> names = new ArrayList<>();
+ for (NormaliserField field : normaliserFields) {
+ if (!names.contains(field.getNorm())) {
+ names.add(field.getNorm());
+ }
+ }
+ return names;
+ }
+
+ /**
+ * The normalised fields, in output order.
+ *
+ * <p>A normalised field is filled from a different input field on every row
it writes, so it has
+ * to describe all of them. When they share a type it takes that type, from
the first of them, as
+ * it always has. When they do not, it is a String, and the transform writes
the text of each
+ * value into it: a field that is declared one type and holds another on
some rows fails the first
+ * transform that serializes or renders it. See issue #3636.
+ *
+ * @param inputRowMeta the fields entering the transform
+ * @return one value metadata per name of {@link
#getNormalisedFieldNames()}, in the same order
+ * @throws HopTransformException when the first input field of a normalised
field is missing
+ */
+ public List<IValueMeta> getNormalisedValueMetas(IRowMeta inputRowMeta)
+ throws HopTransformException {
+ List<IValueMeta> valueMetas = new ArrayList<>();
+ for (String normName : getNormalisedFieldNames()) {
+ List<NormaliserField> fields = getFieldsOf(normName);
+ IValueMeta first = inputRowMeta.searchValueMeta(fields.get(0).getName());
+ if (first == null) {
+ throw new HopTransformException(
+ BaseMessages.getString(
+ PKG, "NormaliserMeta.Exception.UnableToFindField",
fields.get(0).getName()));
+ }
+ boolean sameType = true;
+ boolean sameStorage = true;
+ for (NormaliserField field : fields) {
+ IValueMeta source = inputRowMeta.searchValueMeta(field.getName());
+ if (source != null) {
+ sameType &= source.getType() == first.getType();
+ sameStorage &= source.getStorageType() == first.getStorageType();
+ }
+ }
+
+ IValueMeta v;
+ if (!sameType) {
+ v = new ValueMetaString(normName);
+ } else {
+ v = first.clone();
+ if (!sameStorage) {
+ v.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL);
+ v.setStorageMetadata(null);
+ }
+ }
+ v.setName(normName);
+ valueMetas.add(v);
+ }
+ return valueMetas;
+ }
+
@Override
public void getFields(
IRowMeta row,
@@ -122,23 +184,17 @@ public class NormaliserMeta extends
BaseTransformMeta<Normaliser, NormaliserData
IHopMetadataProvider metadataProvider)
throws HopTransformException {
- // Get a unique list of the occurrences of the type
- //
- List<String> normOcc = new ArrayList<>();
- List<String> fieldOcc = new ArrayList<>();
int maxlen = 0;
- // for (int i = 0; i < normaliserFields.length; i++) {
for (NormaliserField field : normaliserFields) {
- if (!normOcc.contains(field.getNorm())) {
- normOcc.add(field.getNorm());
- fieldOcc.add(field.getName());
- }
-
if (field.getValue().length() > maxlen) {
maxlen = field.getValue().length();
}
}
+ // Take the normalised fields from the input before adding anything to it.
+ //
+ List<IValueMeta> normalisedValueMetas = getNormalisedValueMetas(row);
+
// Then add the type field!
//
IValueMeta typefieldValue = new ValueMetaString(typeField);
@@ -146,21 +202,9 @@ public class NormaliserMeta extends
BaseTransformMeta<Normaliser, NormaliserData
typefieldValue.setLength(maxlen);
row.addValueMeta(typefieldValue);
- // Loop over the distinct list of fieldNorm[i]
// Add the new fields that need to be created.
- // Use the same data type as the original fieldname...
//
- for (int i = 0; i < normOcc.size(); i++) {
- String normname = normOcc.get(i);
- String fieldname = fieldOcc.get(i);
- IValueMeta v = row.searchValueMeta(fieldname);
- if (v != null) {
- v = v.clone();
- } else {
- throw new HopTransformException(
- BaseMessages.getString(PKG,
"NormaliserMeta.Exception.UnableToFindField", fieldname));
- }
- v.setName(normname);
+ for (IValueMeta v : normalisedValueMetas) {
v.setOrigin(name);
row.addValueMeta(v);
}
@@ -226,6 +270,16 @@ public class NormaliserMeta extends
BaseTransformMeta<Normaliser, NormaliserData
transformMeta);
}
remarks.add(cr);
+
+ for (String normName : getNormalisedFieldNames()) {
+ if (hasMixedTypes(prev, normName)) {
+ remarks.add(
+ new CheckResult(
+ ICheckResult.TYPE_RESULT_WARNING,
+ BaseMessages.getString(PKG,
"NormaliserMeta.CheckResult.MixedTypes", normName),
+ transformMeta));
+ }
+ }
} else {
errorMessage =
BaseMessages.getString(
@@ -235,6 +289,11 @@ public class NormaliserMeta extends
BaseTransformMeta<Normaliser, NormaliserData
remarks.add(cr);
}
+ String duplicate = getDuplicateMapping();
+ if (duplicate != null) {
+ remarks.add(new CheckResult(ICheckResult.TYPE_RESULT_ERROR, duplicate,
transformMeta));
+ }
+
// See if we have input streams leading to this transform!
if (input.length > 0) {
cr =
@@ -252,4 +311,48 @@ public class NormaliserMeta extends
BaseTransformMeta<Normaliser, NormaliserData
remarks.add(cr);
}
}
+
+ /** The fields filling a normalised field, in the order they are listed. */
+ private List<NormaliserField> getFieldsOf(String normName) {
+ return normaliserFields.stream()
+ .filter(field -> Objects.equals(normName, field.getNorm()))
+ .toList();
+ }
+
+ /** True when the input fields filling this normalised field are not all of
one type. */
+ private boolean hasMixedTypes(IRowMeta inputRowMeta, String normName) {
+ return getFieldsOf(normName).stream()
+ .map(field -> inputRowMeta.searchValueMeta(field.getName()))
+ .filter(Objects::nonNull)
+ .map(IValueMeta::getType)
+ .distinct()
+ .count()
+ > 1;
+ }
+
+ /**
+ * Two input fields filling the same normalised field on the same row leave
no room for one of
+ * them.
+ *
+ * @return a description of the first such pair, or null when there is none
+ */
+ public String getDuplicateMapping() {
+ for (int i = 0; i < normaliserFields.size(); i++) {
+ NormaliserField one = normaliserFields.get(i);
+ for (int j = i + 1; j < normaliserFields.size(); j++) {
+ NormaliserField other = normaliserFields.get(j);
+ if (Objects.equals(one.getValue(), other.getValue())
+ && Objects.equals(one.getNorm(), other.getNorm())) {
+ return BaseMessages.getString(
+ PKG,
+ "NormaliserMeta.CheckResult.DuplicateMapping",
+ one.getName(),
+ other.getName(),
+ one.getNorm(),
+ one.getValue());
+ }
+ }
+ }
+ return null;
+ }
}
diff --git
a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_en_US.properties
b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_en_US.properties
index ce6e461537..04e13833c1 100644
---
a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_en_US.properties
+++
b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_en_US.properties
@@ -31,7 +31,9 @@ NormaliserDialog.TransformName.Label=Transform name
NormaliserDialog.TypeField.Label=Type field
NormaliserMeta.CheckResult.AllFieldsFound=All fields to normalise are found in
the input stream.
NormaliserMeta.CheckResult.CouldNotReadFieldsFromPreviousTransform=Couldn''t
read fields from the previous transform.
+NormaliserMeta.CheckResult.DuplicateMapping=Fields {0} and {1} both fill
normalised field {2} for type {3}. Every row has room for only one of them.
NormaliserMeta.CheckResult.FieldsNotFound=Fields to normalise, not found in
input stream:
+NormaliserMeta.CheckResult.MixedTypes=Normalised field {0} is filled from
fields of different types, so it is a String holding the text of each value.
NormaliserMeta.CheckResult.NoInputReceivedError=No input received from other
transforms\!
NormaliserMeta.CheckResult.TransformReceivingFieldsOK=Transform is connected
to previous one, receiving {0} fields
NormaliserMeta.CheckResult.TransformReceivingInfoOK=Transform is receiving
info from other transforms.
diff --git
a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaTest.java
b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaTest.java
index 0e4e475d31..be2131f6ea 100644
---
a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaTest.java
+++
b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaTest.java
@@ -16,9 +16,28 @@
*/
package org.apache.hop.pipeline.transforms.normaliser;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+
+import java.util.ArrayList;
+import java.util.List;
import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.ICheckResult;
+import org.apache.hop.core.exception.HopTransformException;
import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaNumber;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.variables.Variables;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transform.TransformSerializationTestUtil;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -45,4 +64,168 @@ class NormaliserMetaTest {
// assertEquals("fieldName", meta.getFields().get(0).getName());
// assertEquals("two", meta.getFields().get(0).getValue());
}
+
+ @Test
+ void aNormalisedFieldOfOneTypeTakesThatTypeFromItsFirstField() throws
Exception {
+ IRowMeta row = new RowMeta();
+ ValueMetaInteger first = new ValueMetaInteger("i1");
+ first.setLength(7);
+ row.addValueMeta(first);
+ row.addValueMeta(new ValueMetaInteger("i2"));
+
+ meta(field("i1", "A", "value"), field("i2", "B", "value"))
+ .getFields(row, "normaliser", null, null, new Variables(), null);
+
+ assertArrayEquals(new String[] {"typefield", "value"},
row.getFieldNames());
+ IValueMeta value = row.getValueMeta(1);
+ assertEquals(IValueMeta.TYPE_INTEGER, value.getType());
+ assertEquals(7, value.getLength());
+ assertEquals("normaliser", value.getOrigin());
+ }
+
+ /** Issue #3636: the first field used to decide, whatever the others were. */
+ @Test
+ void aNormalisedFieldOfMixedTypesIsAString() throws Exception {
+ IRowMeta row = new RowMeta();
+ row.addValueMeta(new ValueMetaInteger("integer"));
+ row.addValueMeta(new ValueMetaNumber("number"));
+ row.addValueMeta(new ValueMetaString("kept"));
+
+ meta(field("integer", "integer", "value"), field("number", "number",
"value"))
+ .getFields(row, "normaliser", null, null, new Variables(), null);
+
+ assertArrayEquals(new String[] {"kept", "typefield", "value"},
row.getFieldNames());
+ assertEquals(IValueMeta.TYPE_STRING, row.getValueMeta(2).getType());
+ }
+
+ @Test
+ void aNormalisedFieldWhoseFirstFieldIsMissingCannotBeDescribed() {
+ IRowMeta row = new RowMeta();
+ row.addValueMeta(new ValueMetaInteger("i2"));
+
+ HopTransformException e =
+ assertThrows(
+ HopTransformException.class,
+ () ->
+ meta(field("i1", "A", "value"), field("i2", "B", "value"))
+ .getFields(row, "normaliser", null, null, new Variables(),
null));
+ assertTrue(e.getMessage().contains("i1"), e.getMessage());
+ }
+
+ /** A later field that is missing is left to the transform to report when it
runs. */
+ @Test
+ void aMissingLaterFieldDoesNotDecideTheType() throws Exception {
+ IRowMeta row = new RowMeta();
+ row.addValueMeta(new ValueMetaInteger("i1"));
+
+ meta(field("i1", "A", "value"), field("missing", "B", "value"))
+ .getFields(row, "normaliser", null, null, new Variables(), null);
+
+ assertEquals(IValueMeta.TYPE_INTEGER,
row.getValueMeta(row.indexOfValue("value")).getType());
+ }
+
+ @Test
+ void checkReportsMissingFieldsAndStillWarnsAboutTheOthers() {
+ IRowMeta prev = new RowMeta();
+ prev.addValueMeta(new ValueMetaInteger("integer"));
+ prev.addValueMeta(new ValueMetaString("text"));
+
+ List<ICheckResult> remarks =
+ check(
+ meta(
+ field("integer", "integer", "value"),
+ field("text", "text", "value"),
+ field("missing", "missing", "value")),
+ prev);
+
+ assertTrue(
+ remarks.stream()
+ .anyMatch(
+ r ->
+ r.getType() == ICheckResult.TYPE_RESULT_ERROR
+ && r.getText().contains("missing")),
+ remarks.toString());
+ assertTrue(
+ remarks.stream().anyMatch(r -> r.getType() ==
ICheckResult.TYPE_RESULT_WARNING),
+ remarks.toString());
+ }
+
+ @Test
+ void checkWarnsAboutAFieldOfMixedTypes() {
+ IRowMeta prev = new RowMeta();
+ prev.addValueMeta(new ValueMetaInteger("integer"));
+ prev.addValueMeta(new ValueMetaString("text"));
+
+ List<ICheckResult> remarks =
+ check(meta(field("integer", "integer", "value"), field("text", "text",
"value")), prev);
+
+ assertTrue(
+ remarks.stream()
+ .anyMatch(
+ r ->
+ r.getType() == ICheckResult.TYPE_RESULT_WARNING
+ && r.getText().contains("value")),
+ remarks.toString());
+ }
+
+ @Test
+ void checkReportsTwoFieldsFillingOneNormalisedFieldOfOneType() {
+ IRowMeta prev = new RowMeta();
+ prev.addValueMeta(new ValueMetaString("a1"));
+ prev.addValueMeta(new ValueMetaString("a2"));
+
+ List<ICheckResult> remarks = check(meta(field("a1", "A", "X"), field("a2",
"A", "X")), prev);
+
+ assertTrue(
+ remarks.stream()
+ .anyMatch(
+ r ->
+ r.getType() == ICheckResult.TYPE_RESULT_ERROR
+ && r.getText().contains("a1")
+ && r.getText().contains("a2")),
+ remarks.toString());
+ }
+
+ @Test
+ void checkIsQuietAboutFieldsOfOneType() {
+ IRowMeta prev = new RowMeta();
+ prev.addValueMeta(new ValueMetaInteger("i1"));
+ prev.addValueMeta(new ValueMetaInteger("i2"));
+
+ List<ICheckResult> remarks = check(meta(field("i1", "A", "X"), field("i2",
"B", "X")), prev);
+
+ assertTrue(
+ remarks.stream().allMatch(r -> r.getType() ==
ICheckResult.TYPE_RESULT_OK),
+ remarks.toString());
+ }
+
+ private static List<ICheckResult> check(NormaliserMeta meta, IRowMeta prev) {
+ List<ICheckResult> remarks = new ArrayList<>();
+ meta.check(
+ remarks,
+ mock(PipelineMeta.class),
+ mock(TransformMeta.class),
+ prev,
+ new String[] {"input"},
+ new String[0],
+ null,
+ new Variables(),
+ null);
+ return remarks;
+ }
+
+ private static NormaliserField field(String name, String type, String norm) {
+ NormaliserField field = new NormaliserField();
+ field.setName(name);
+ field.setValue(type);
+ field.setNorm(norm);
+ return field;
+ }
+
+ private static NormaliserMeta meta(NormaliserField... fields) {
+ NormaliserMeta meta = new NormaliserMeta();
+ meta.setDefault();
+ meta.setNormaliserFields(new ArrayList<>(List.of(fields)));
+ return meta;
+ }
}
diff --git
a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserTest.java
b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserTest.java
index 40c341db93..cae74dc9fa 100644
---
a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserTest.java
+++
b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserTest.java
@@ -14,172 +14,387 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.hop.pipeline.transforms.normaliser;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Date;
import java.util.List;
+import org.apache.hop.core.BlockingRowSet;
import org.apache.hop.core.HopEnvironment;
-import org.apache.hop.core.RowMetaAndData;
import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILoggingObject;
import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowMeta;
import org.apache.hop.core.row.value.ValueMetaDate;
import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaNumber;
import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
+import org.mockito.stubbing.Stubber;
class NormaliserTest {
@RegisterExtension
static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+ private TransformMockHelper<NormaliserMeta, NormaliserData> helper;
+
@BeforeAll
static void before() throws HopException {
HopEnvironment.init();
}
- private NormaliserField[] getTestNormaliserFieldsWiki() {
- NormaliserField[] rtn = new NormaliserField[6];
- rtn[0] = new NormaliserField();
- rtn[0].setName("pr_sl");
- rtn[0].setNorm("Product Sales");
- rtn[0].setValue("Product1"); // Type
-
- rtn[1] = new NormaliserField();
- rtn[1].setName("pr1_nr");
- rtn[1].setNorm("Product Number");
- rtn[1].setValue("Product1");
-
- rtn[2] = new NormaliserField();
- rtn[2].setName("pr2_sl");
- rtn[2].setNorm("Product Sales");
- rtn[2].setValue("Product2");
-
- rtn[3] = new NormaliserField();
- rtn[3].setName("pr2_nr");
- rtn[3].setNorm("Product Number");
- rtn[3].setValue("Product2");
-
- rtn[4] = new NormaliserField();
- rtn[4].setName("pr3_sl");
- rtn[4].setNorm("Product Sales");
- rtn[4].setValue("Product3");
-
- rtn[5] = new NormaliserField();
- rtn[5].setName("pr3_nr");
- rtn[5].setNorm("Product Number");
- rtn[5].setValue("Product3");
-
- return rtn;
- }
-
- private List<RowMetaAndData> getExpectedWikiOutputRowMetaAndData() {
- final Date theDate = new Date(103, 01, 01);
- List<RowMetaAndData> list = new ArrayList<>();
- IRowMeta rm = new RowMeta();
- rm.addValueMeta(new ValueMetaDate("DATE"));
- rm.addValueMeta(new ValueMetaString("Type"));
- rm.addValueMeta(new ValueMetaInteger("Product Sales"));
- rm.addValueMeta(new ValueMetaInteger("Product Number"));
- Object[] row = new Object[4];
- row[0] = theDate;
- row[1] = "Product1";
- row[2] = 100;
- row[3] = 5;
- list.add(new RowMetaAndData(rm, row));
-
- row = new Object[4];
- row[0] = theDate;
- row[1] = "Product2";
- row[2] = 250;
- row[3] = 10;
- list.add(new RowMetaAndData(rm, row));
-
- row = new Object[4];
- row[0] = theDate;
- row[1] = "Product3";
- row[2] = 150;
- row[3] = 4;
- list.add(new RowMetaAndData(rm, row));
- return list;
- }
-
- private List<RowMetaAndData> getWikiInputRowMetaAndData() {
- List<RowMetaAndData> list = new ArrayList<>();
- Object[] row = new Object[7];
- IRowMeta rm = new RowMeta();
- rm.addValueMeta(new ValueMetaDate("DATE"));
- row[0] = new Date(103, 01, 01);
- rm.addValueMeta(new ValueMetaInteger("PR1_NR"));
- row[1] = 5;
- rm.addValueMeta(new ValueMetaInteger("PR_SL"));
- row[2] = 100;
- rm.addValueMeta(new ValueMetaInteger("PR2_NR"));
- row[3] = 10;
- rm.addValueMeta(new ValueMetaInteger("PR2_SL"));
- row[4] = 250;
- rm.addValueMeta(new ValueMetaInteger("PR3_NR"));
- row[5] = 4;
- rm.addValueMeta(new ValueMetaInteger("PR3_SL"));
- row[6] = 150;
- list.add(new RowMetaAndData(rm, row));
- return list;
- }
-
- private void checkResults(List<RowMetaAndData> expectedOutput,
List<RowMetaAndData> outputList) {
- assertEquals(expectedOutput.size(), outputList.size());
- for (int i = 0; i < outputList.size(); i++) {
- RowMetaAndData aRowMetaAndData = outputList.get(i);
- RowMetaAndData expectedRowMetaAndData = expectedOutput.get(i);
- IRowMeta rowMeta = aRowMetaAndData.getRowMeta();
- IRowMeta expectedRowMeta = expectedRowMetaAndData.getRowMeta();
- String[] fields = rowMeta.getFieldNames();
- String[] expectedFields = expectedRowMeta.getFieldNames();
- assertEquals(expectedFields.length, fields.length);
- assertArrayEquals(expectedFields, fields);
- Object[] aRow = aRowMetaAndData.getData();
- Object[] expectedRow = expectedRowMetaAndData.getData();
- assertEquals(expectedRow.length, aRow.length);
- assertArrayEquals(expectedRow, aRow);
+ @BeforeEach
+ void setUp() {
+ helper =
+ new TransformMockHelper<>("Row normaliser", NormaliserMeta.class,
NormaliserData.class);
+ when(helper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(helper.iLogChannel);
+ when(helper.pipeline.isRunning()).thenReturn(true);
+ }
+
+ @AfterEach
+ void tearDown() {
+ helper.cleanUp();
+ }
+
+ /**
+ * The example the transform was written for:
+ *
+ * <pre>
+ * DATE PR1_NR PR_SL PR2_NR PR2_SL PR3_NR PR3_SL
+ * 20030101 5 100 10 250 4 150
+ * </pre>
+ *
+ * becomes one row per product, with its sales and number.
+ */
+ @Test
+ void productsAreNormalisedIntoOneRowEach() throws Exception {
+ Date date = new Date(103, 0, 1);
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaDate("DATE"));
+ input.addValueMeta(new ValueMetaInteger("PR1_NR"));
+ input.addValueMeta(new ValueMetaInteger("PR_SL"));
+ input.addValueMeta(new ValueMetaInteger("PR2_NR"));
+ input.addValueMeta(new ValueMetaInteger("PR2_SL"));
+ input.addValueMeta(new ValueMetaInteger("PR3_NR"));
+ input.addValueMeta(new ValueMetaInteger("PR3_SL"));
+
+ NormaliserMeta meta =
+ meta(
+ "Type",
+ field("PR_SL", "Product1", "Product Sales"),
+ field("PR1_NR", "Product1", "Product Number"),
+ field("PR2_SL", "Product2", "Product Sales"),
+ field("PR2_NR", "Product2", "Product Number"),
+ field("PR3_SL", "Product3", "Product Sales"),
+ field("PR3_NR", "Product3", "Product Number"));
+
+ Output output = normalise(meta, input, date, 5L, 100L, 10L, 250L, 4L,
150L);
+
+ assertArrayEquals(
+ new String[] {"DATE", "Type", "Product Sales", "Product Number"},
+ output.rowMeta.getFieldNames());
+ assertArrayEquals(new Object[] {date, "Product1", 100L, 5L},
output.rows.get(0));
+ assertArrayEquals(new Object[] {date, "Product2", 250L, 10L},
output.rows.get(1));
+ assertArrayEquals(new Object[] {date, "Product3", 150L, 4L},
output.rows.get(2));
+ }
+
+ /** When every field filling a normalised field has one type, the values go
through untouched. */
+ @Test
+ void fieldsOfOneTypeKeepThatTypeAndTheirValues() throws Exception {
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaString("a"));
+ input.addValueMeta(new ValueMetaString("b"));
+ String a = "first";
+ String b = "second";
+
+ Output output =
+ normalise(meta("type", field("a", "A", "value"), field("b", "B",
"value")), input, a, b);
+
+ assertEquals(IValueMeta.TYPE_STRING,
output.rowMeta.getValueMeta(1).getType());
+ assertSame(a, output.rows.get(0)[1]);
+ assertSame(b, output.rows.get(1)[1]);
+ }
+
+ /**
+ * The example from issue #3636: a String, an Integer and a Number
normalised into one field. That
+ * field used to be declared a String and carry a Long and a Double, which
is fine until something
+ * writes the row out: a sort that spills to disk, for one.
+ */
+ @Test
+ void fieldsOfDifferentTypesFillAStringWithTheTextOfEachValue() throws
Exception {
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaString("text"));
+ input.addValueMeta(new ValueMetaInteger("integer"));
+ ValueMetaNumber number = new ValueMetaNumber("number");
+ number.setConversionMask("0.0");
+ number.setDecimalSymbol(".");
+ input.addValueMeta(number);
+
+ NormaliserMeta meta =
+ meta(
+ "typefield",
+ field("text", "text", "value"),
+ field("integer", "integer", "value"),
+ field("number", "number", "value"));
+
+ Output output = normalise(meta, input, "a", 1L, 1.2);
+
+ IValueMeta value = output.rowMeta.getValueMeta(1);
+ assertEquals(IValueMeta.TYPE_STRING, value.getType());
+ assertEquals(List.of("a", "1", "1.2"), output.rows.stream().map(row ->
row[1]).toList());
+
+ // What the sort's temporary file does with every row.
+ DataOutputStream out = new DataOutputStream(new ByteArrayOutputStream());
+ for (Object[] row : output.rows) {
+ output.rowMeta.writeData(out, row);
}
}
- // @Test
- void testNormaliserProcessRowsWikiData() throws Exception {
- // We should have 1 row as input to the normaliser and 3 rows as output to
the normaliser with
- // the data
- //
- // Data input looks like this:
- //
- // DATE PR1_NR PR_SL PR2_NR PR2_SL PR3_NR PR3_SL
- // 2003010 5 100 10 250 4 150
- //
- // Data output looks like this:
- //
- // DATE Type Product Sales Product Number
- // 2003010 Product1 100 5
- // 2003010 Product2 250 10
- // 2003010 Product3 150 4
- //
-
- // final String transformName = "Row Normaliser";
- // NormaliserMeta transformMeta = new NormaliserMeta();
- // transformMeta.setDefault();
- // transformMeta.setNormaliserFields( getTestNormaliserFieldsWiki() );
- // transformMeta.setTypeField( "Type" );
-
- // PipelineMeta pipelineMeta =
PipelineTestFactory.generateTestTransformation( null,
- // transformMeta, transformName );
- // List<RowMetaAndData> inputList = getWikiInputRowMetaAndData();
- // List<RowMetaAndData> outputList =
PipelineTestFactory.executeTestTransformation(
- // pipelineMeta, PipelineTestFactory.INJECTOR_TRANSFORMNAME, transformName,
- // PipelineTestFactory.DUMMY_TRANSFORMNAME, inputList );
- // List<RowMetaAndData> expectedOutput =
this.getExpectedWikiOutputRowMetaAndData();
- // checkResults( expectedOutput, outputList );
+ /** The same holds when the first field is not the String: the field used to
be an Integer. */
+ @Test
+ void theFirstFieldNoLongerDecidesTheTypeOfTheOthers() throws Exception {
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaInteger("integer"));
+ input.addValueMeta(new ValueMetaString("text"));
+
+ Output output =
+ normalise(
+ meta("type", field("integer", "integer", "value"), field("text",
"text", "value")),
+ input,
+ 7L,
+ "seven");
+
+ assertEquals(IValueMeta.TYPE_STRING,
output.rowMeta.getValueMeta(1).getType());
+ assertEquals(List.of("7", "seven"), output.rows.stream().map(row ->
row[1]).toList());
+ }
+
+ /** Nulls stay null, whether they are copied or converted. */
+ @Test
+ void nullsStayNull() throws Exception {
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaString("text"));
+ input.addValueMeta(new ValueMetaInteger("integer"));
+
+ Output output =
+ normalise(
+ meta("type", field("text", "text", "value"), field("integer",
"integer", "value")),
+ input,
+ null,
+ null);
+
+ assertNull(output.rows.get(0)[1]);
+ assertNull(output.rows.get(1)[1]);
+ }
+
+ /**
+ * A value goes into the normalised field it names, whatever order the
fields are listed in. Type
+ * B's fields are listed Y before X, the other way round from type A's, and
used to land in each
+ * other's place.
+ */
+ @Test
+ void valuesGoIntoTheFieldTheyNameWhateverTheOrderTheyAreListedIn() throws
Exception {
+ IRowMeta input = new RowMeta();
+ for (String name : new String[] {"a1", "a2", "b1", "b2"}) {
+ input.addValueMeta(new ValueMetaString(name));
+ }
+
+ NormaliserMeta meta =
+ meta(
+ "type",
+ field("a1", "A", "X"),
+ field("b2", "B", "Y"),
+ field("b1", "B", "X"),
+ field("a2", "A", "Y"));
+
+ Output output = normalise(meta, input, "a1", "a2", "b1", "b2");
+
+ assertArrayEquals(new String[] {"type", "X", "Y"},
output.rowMeta.getFieldNames());
+ assertArrayEquals(new Object[] {"A", "a1", "a2"}, output.rows.get(0));
+ assertArrayEquals(new Object[] {"B", "b1", "b2"}, output.rows.get(1));
+ }
+
+ /** A type that has no field for one of the normalised fields leaves it
empty. */
+ @Test
+ void aTypeWithoutAFieldForANormalisedFieldLeavesItNull() throws Exception {
+ IRowMeta input = new RowMeta();
+ for (String name : new String[] {"a1", "a2", "b2"}) {
+ input.addValueMeta(new ValueMetaString(name));
+ }
+
+ NormaliserMeta meta =
+ meta("type", field("a1", "A", "X"), field("a2", "A", "Y"), field("b2",
"B", "Y"));
+
+ Output output = normalise(meta, input, "a1", "a2", "b2");
+
+ assertArrayEquals(new Object[] {"A", "a1", "a2"}, output.rows.get(0));
+ assertArrayEquals(new Object[] {"B", null, "b2"}, output.rows.get(1));
+ }
+
+ /** Two fields of one type filling the same normalised field cannot both fit
on its row. */
+ @Test
+ void twoFieldsFillingTheSameNormalisedFieldOfOneTypeAreRefused() throws
Exception {
+ IRowMeta input = new RowMeta();
+ for (String name : new String[] {"a1", "a2"}) {
+ input.addValueMeta(new ValueMetaString(name));
+ }
+
+ NormaliserMeta meta = meta("type", field("a1", "A", "X"), field("a2", "A",
"X"));
+
+ HopException e = assertThrows(HopException.class, () -> normalise(meta,
input, "a1", "a2"));
+ assertTrue(e.getMessage().contains("a1"), e.getMessage());
+ assertTrue(e.getMessage().contains("a2"), e.getMessage());
+ }
+
+ /** Lazy conversion hands over binary strings; a field of both storages gets
normal ones. */
+ @Test
+ void fieldsOfOneTypeInDifferentStoragesFillAFieldOfNormalStorage() throws
Exception {
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaString("plain"));
+ ValueMetaString lazy = new ValueMetaString("lazy");
+ lazy.setStorageType(IValueMeta.STORAGE_TYPE_BINARY_STRING);
+ ValueMetaString storage = new ValueMetaString("lazy");
+ lazy.setStorageMetadata(storage);
+ input.addValueMeta(lazy);
+
+ Output output =
+ normalise(
+ meta("type", field("plain", "plain", "value"), field("lazy",
"lazy", "value")),
+ input,
+ "plain value",
+ "lazy value".getBytes(StandardCharsets.UTF_8));
+
+ IValueMeta value = output.rowMeta.getValueMeta(1);
+ assertEquals(IValueMeta.TYPE_STRING, value.getType());
+ assertEquals(IValueMeta.STORAGE_TYPE_NORMAL, value.getStorageType());
+ assertEquals(
+ List.of("plain value", "lazy value"), output.rows.stream().map(row ->
row[1]).toList());
+ }
+
+ /** The placements worked out on the first row serve every row after it. */
+ @Test
+ void everyRowIsNormalisedTheSameWay() throws Exception {
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaString("text"));
+ input.addValueMeta(new ValueMetaInteger("integer"));
+
+ Output output =
+ normaliseRows(
+ meta("type", field("text", "text", "value"), field("integer",
"integer", "value")),
+ input,
+ new Object[] {"a", 1L},
+ new Object[] {"b", 2L});
+
+ assertEquals(List.of("a", "1", "b", "2"), output.rows.stream().map(row ->
row[1]).toList());
+ }
+
+ /** A field that is not in the input stops the transform with an error. */
+ @Test
+ void aFieldMissingFromTheInputStopsTheTransform() throws Exception {
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaString("text"));
+
+ NormaliserMeta meta =
+ meta("type", field("text", "text", "value"), field("missing",
"missing", "value"));
+ when(helper.transformMeta.getTransform()).thenReturn(meta);
+ Normaliser transform =
+ new Normaliser(
+ helper.transformMeta,
+ meta,
+ new NormaliserData(),
+ 0,
+ helper.pipelineMeta,
+ helper.pipeline);
+ transform.init();
+ transform.setInputRowMeta(input);
+ transform.setOutputRowSets(Collections.singletonList(new
BlockingRowSet(10)));
+ Normaliser spied = spy(transform);
+ doReturn(new Object[] {"a"}).when(spied).getRow();
+
+ assertFalse(spied.processRow());
+ assertEquals(1, spied.getErrors());
+ }
+
+ private static NormaliserField field(String name, String type, String norm) {
+ NormaliserField field = new NormaliserField();
+ field.setName(name);
+ field.setValue(type);
+ field.setNorm(norm);
+ return field;
+ }
+
+ private static NormaliserMeta meta(String typeField, NormaliserField...
fields) {
+ NormaliserMeta meta = new NormaliserMeta();
+ meta.setTypeField(typeField);
+ meta.setNormaliserFields(new ArrayList<>(List.of(fields)));
+ return meta;
+ }
+
+ private record Output(IRowMeta rowMeta, List<Object[]> rows) {}
+
+ /** Runs one input row through the transform and collects what it writes. */
+ private Output normalise(NormaliserMeta meta, IRowMeta input, Object... row)
throws Exception {
+ return normaliseRows(meta, input, row);
+ }
+
+ /** Runs input rows through the transform and collects what it writes. */
+ private Output normaliseRows(NormaliserMeta meta, IRowMeta input,
Object[]... rows)
+ throws Exception {
+ when(helper.transformMeta.getTransform()).thenReturn(meta);
+ Normaliser transform =
+ new Normaliser(
+ helper.transformMeta,
+ meta,
+ new NormaliserData(),
+ 0,
+ helper.pipelineMeta,
+ helper.pipeline);
+ transform.init();
+ transform.setInputRowMeta(input);
+ BlockingRowSet rowSet = new BlockingRowSet(100);
+ transform.setOutputRowSets(Collections.singletonList(rowSet));
+
+ Normaliser spied = spy(transform);
+ Stubber stubber = doReturn(rows[0]);
+ for (int i = 1; i < rows.length; i++) {
+ stubber = stubber.doReturn(rows[i]);
+ }
+ stubber.doReturn(null).when(spied).getRow();
+
+ for (int i = 0; i < rows.length; i++) {
+ assertTrue(spied.processRow());
+ }
+ assertFalse(spied.processRow());
+
+ List<Object[]> output = new ArrayList<>();
+ Object[] written;
+ while ((written = rowSet.getRowImmediate()) != null) {
+ output.add(written);
+ }
+ return new Output(rowSet.getRowMeta(), output);
}
}
