This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 878cafbc33 Improved Split fields: option to keep the field to split
and append new fields (#7857)
878cafbc33 is described below
commit 878cafbc338833bf3294ec0a6ba77cd848c04702
Author: Lance <[email protected]>
AuthorDate: Wed Aug 12 22:32:32 2026 +0800
Improved Split fields: option to keep the field to split and append new
fields (#7857)
Signed-off-by: lance <[email protected]>
---
.../pages/pipeline/transforms/splitfields.adoc | 1 +
...miter.hpl => 0003-field-splitter-delimiter.hpl} | 3 +-
...it-fields-id.hpl => 0003-field-splitter-id.hpl} | 3 +-
...iter.hpl => 0003-field-splitter-keep-field.hpl} | 9 +-
.../datasets/golden-field-splitter-keep-field.csv | 6 +
.../transforms/main-0003-field-splitter.hwf | 28 ++
.../transforms/main-0054-split-fields.hwf | 82 ------
.../dataset/golden-field-splitter-keep-field.json | 64 ++++
...son => 0003-field-splitter-delimiter UNIT.json} | 6 +-
... UNIT.json => 0003-field-splitter-id UNIT.json} | 6 +-
...on => 0003-field-splitter-keep-field UNIT.json} | 22 +-
.../transforms/fieldsplitter/FieldSplitter.java | 34 ++-
.../fieldsplitter/FieldSplitterDialog.java | 40 ++-
.../fieldsplitter/FieldSplitterMeta.java | 324 ++-------------------
.../messages/messages_en_US.properties | 3 +
.../messages/messages_zh_CN.properties | 3 +
.../FieldSplitterKeepSplitFieldTest.java | 147 ++++++++++
.../fieldsplitter/FieldSplitterMetaTest.java | 3 +
18 files changed, 373 insertions(+), 411 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/splitfields.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/splitfields.adoc
index 62384e15ee..45c2b8f5d8 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/splitfields.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/splitfields.adoc
@@ -53,6 +53,7 @@ Special characters (e.g. CHAR ASCII HEX01) can be set with
the format $[value],
For example with a comma (,) delimiter: ```aaaa,"123,456",ccc``` can be
resolved to 3 fields using enclosure ```"```.
| Escape string|To include delimiter characters in values sometimes an escape
string is used like backslash, double backslash and so on.
For example with a comma (,) delimiter: ```aaaa,123\\,456,ccc``` can be
resolved to 3 fields using escape character ```\\```.
+|Keep the field to split|When enabled, the original field is kept and the new
fields are appended at the end of the field list. When disabled (default), the
original field is replaced in place by the new fields.
|Fields table|This table is where you define the properties for each new field
created by the split.
For each new field, you must define the field name, data type, and other
properties.
|===
diff --git a/integration-tests/transforms/0054-split-fields-delimiter.hpl
b/integration-tests/transforms/0003-field-splitter-delimiter.hpl
similarity index 98%
copy from integration-tests/transforms/0054-split-fields-delimiter.hpl
copy to integration-tests/transforms/0003-field-splitter-delimiter.hpl
index 20701199ca..1a0eb250e0 100644
--- a/integration-tests/transforms/0054-split-fields-delimiter.hpl
+++ b/integration-tests/transforms/0003-field-splitter-delimiter.hpl
@@ -19,7 +19,7 @@ limitations under the License.
-->
<pipeline>
<info>
- <name>0054-split-fields-delimiter</name>
+ <name>0003-field-splitter-delimiter</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@@ -64,6 +64,7 @@ limitations under the License.
<delimiter>,</delimiter>
<enclosure/>
<escape_string/>
+ <keep_split_field>N</keep_split_field>
<fields>
<field>
<name>sales1</name>
diff --git a/integration-tests/transforms/0054-split-fields-id.hpl
b/integration-tests/transforms/0003-field-splitter-id.hpl
similarity index 98%
rename from integration-tests/transforms/0054-split-fields-id.hpl
rename to integration-tests/transforms/0003-field-splitter-id.hpl
index 2b8ee4de0f..7e3ac978d4 100644
--- a/integration-tests/transforms/0054-split-fields-id.hpl
+++ b/integration-tests/transforms/0003-field-splitter-id.hpl
@@ -19,7 +19,7 @@ limitations under the License.
-->
<pipeline>
<info>
- <name>0054-split-fields-id</name>
+ <name>0003-field-splitter-id</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@@ -64,6 +64,7 @@ limitations under the License.
<delimiter>,</delimiter>
<enclosure/>
<escape_string/>
+ <keep_split_field>N</keep_split_field>
<fields>
<field>
<name>a</name>
diff --git a/integration-tests/transforms/0054-split-fields-delimiter.hpl
b/integration-tests/transforms/0003-field-splitter-keep-field.hpl
similarity index 94%
rename from integration-tests/transforms/0054-split-fields-delimiter.hpl
rename to integration-tests/transforms/0003-field-splitter-keep-field.hpl
index 20701199ca..1024095344 100644
--- a/integration-tests/transforms/0054-split-fields-delimiter.hpl
+++ b/integration-tests/transforms/0003-field-splitter-keep-field.hpl
@@ -19,9 +19,9 @@ limitations under the License.
-->
<pipeline>
<info>
- <name>0054-split-fields-delimiter</name>
+ <name>0003-field-splitter-keep-field</name>
<name_sync_with_filename>Y</name_sync_with_filename>
- <description/>
+ <description>Keep the field to split and append new fields at the
end</description>
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
@@ -31,9 +31,9 @@ limitations under the License.
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
- <created_date>2022/11/27 16:28:58.618</created_date>
+ <created_date>2026/08/08 16:00:00.000</created_date>
<modified_user>-</modified_user>
- <modified_date>2022/11/27 16:28:58.618</modified_date>
+ <modified_date>2026/08/08 16:00:00.000</modified_date>
</info>
<notepads>
</notepads>
@@ -64,6 +64,7 @@ limitations under the License.
<delimiter>,</delimiter>
<enclosure/>
<escape_string/>
+ <keep_split_field>Y</keep_split_field>
<fields>
<field>
<name>sales1</name>
diff --git
a/integration-tests/transforms/datasets/golden-field-splitter-keep-field.csv
b/integration-tests/transforms/datasets/golden-field-splitter-keep-field.csv
new file mode 100644
index 0000000000..f619243110
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-field-splitter-keep-field.csv
@@ -0,0 +1,6 @@
+id,value,value2,sales1,sales2,sales3,sales4
+1,"500,300,200,100",v1,500,300,200,100
+2,"600,400,300,200",v2,600,400,300,200
+3,"700,500,400,300",v3,700,500,400,300
+4,"800,600,500,400",v4,800,600,500,400
+5,"900,700,600,500",v5,900,700,600,500
diff --git a/integration-tests/transforms/main-0003-field-splitter.hwf
b/integration-tests/transforms/main-0003-field-splitter.hwf
index f80d75676d..db95839c04 100644
--- a/integration-tests/transforms/main-0003-field-splitter.hwf
+++ b/integration-tests/transforms/main-0003-field-splitter.hwf
@@ -141,6 +141,27 @@ limitations under the License.
<yloc>64</yloc>
<attributes_hac/>
</action>
+ <action>
+ <name>Run Pipeline Unit Tests</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0003-field-splitter-delimiter UNIT</name>
+ </test_name>
+ <test_name>
+ <name>0003-field-splitter-id UNIT</name>
+ </test_name>
+ <test_name>
+ <name>0003-field-splitter-keep-field UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>800</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
</actions>
<hops>
<hop>
@@ -164,6 +185,13 @@ limitations under the License.
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
+ <hop>
+ <from>0003-field-splitter-using-ids</from>
+ <to>Run Pipeline Unit Tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
</hops>
<notepads>
</notepads>
diff --git a/integration-tests/transforms/main-0054-split-fields.hwf
b/integration-tests/transforms/main-0054-split-fields.hwf
deleted file mode 100644
index edf20da32e..0000000000
--- a/integration-tests/transforms/main-0054-split-fields.hwf
+++ /dev/null
@@ -1,82 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
--->
-<workflow>
- <name>main-0054-split-fields</name>
- <name_sync_with_filename>Y</name_sync_with_filename>
- <description/>
- <extended_description/>
- <workflow_version/>
- <created_user>-</created_user>
- <created_date>2022/11/27 16:27:52.087</created_date>
- <modified_user>-</modified_user>
- <modified_date>2022/11/27 16:27:52.087</modified_date>
- <parameters>
- </parameters>
- <actions>
- <action>
- <name>Start</name>
- <description/>
- <type>SPECIAL</type>
- <attributes/>
- <DayOfMonth>1</DayOfMonth>
- <hour>12</hour>
- <intervalMinutes>60</intervalMinutes>
- <intervalSeconds>0</intervalSeconds>
- <minutes>0</minutes>
- <repeat>N</repeat>
- <schedulerType>0</schedulerType>
- <weekDay>1</weekDay>
- <parallel>N</parallel>
- <xloc>80</xloc>
- <yloc>64</yloc>
- <attributes_hac/>
- </action>
- <action>
- <name>Run Pipeline Unit Tests</name>
- <description/>
- <type>RunPipelineTests</type>
- <attributes/>
- <test_names>
- <test_name>
- <name>0054-split-fields-delimiter UNIT</name>
- </test_name>
- <test_name>
- <name>0054-split-fields-id UNIT</name>
- </test_name>
- </test_names>
- <parallel>N</parallel>
- <xloc>288</xloc>
- <yloc>64</yloc>
- <attributes_hac/>
- </action>
- </actions>
- <hops>
- <hop>
- <from>Start</from>
- <to>Run Pipeline Unit Tests</to>
- <enabled>Y</enabled>
- <evaluation>Y</evaluation>
- <unconditional>Y</unconditional>
- </hop>
- </hops>
- <notepads>
- </notepads>
- <attributes/>
-</workflow>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-field-splitter-keep-field.json
b/integration-tests/transforms/metadata/dataset/golden-field-splitter-keep-field.json
new file mode 100644
index 0000000000..6024f6fa8b
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-field-splitter-keep-field.json
@@ -0,0 +1,64 @@
+{
+ "base_filename": "golden-field-splitter-keep-field.csv",
+ "name": "golden-field-splitter-keep-field",
+ "description": "Keep the original field and append split fields at the end",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "id",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "value",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "value2",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "sales1",
+ "field_format": "#"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "sales2",
+ "field_format": "#"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "sales3",
+ "field_format": "#"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "sales4",
+ "field_format": "#"
+ }
+ ],
+ "folder_name": ""
+}
diff --git
a/integration-tests/transforms/metadata/unit-test/0054-split-fields-delimiter
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0003-field-splitter-delimiter
UNIT.json
similarity index 89%
copy from
integration-tests/transforms/metadata/unit-test/0054-split-fields-delimiter
UNIT.json
copy to
integration-tests/transforms/metadata/unit-test/0003-field-splitter-delimiter
UNIT.json
index e11735186d..cdc6e9faf5 100644
---
a/integration-tests/transforms/metadata/unit-test/0054-split-fields-delimiter
UNIT.json
+++
b/integration-tests/transforms/metadata/unit-test/0003-field-splitter-delimiter
UNIT.json
@@ -39,10 +39,10 @@
}
],
"input_data_sets": [],
- "name": "0054-split-fields-delimiter UNIT",
+ "name": "0003-field-splitter-delimiter UNIT",
"description": "",
"persist_filename": "",
"trans_test_tweaks": [],
- "pipeline_filename": "./0054-split-fields-delimiter.hpl",
+ "pipeline_filename": "./0003-field-splitter-delimiter.hpl",
"test_type": "UNIT_TEST"
-}
\ No newline at end of file
+}
diff --git
a/integration-tests/transforms/metadata/unit-test/0054-split-fields-id
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0003-field-splitter-id
UNIT.json
similarity index 90%
rename from
integration-tests/transforms/metadata/unit-test/0054-split-fields-id UNIT.json
rename to
integration-tests/transforms/metadata/unit-test/0003-field-splitter-id UNIT.json
index 46a4d23279..b0e54c6335 100644
--- a/integration-tests/transforms/metadata/unit-test/0054-split-fields-id
UNIT.json
+++ b/integration-tests/transforms/metadata/unit-test/0003-field-splitter-id
UNIT.json
@@ -39,10 +39,10 @@
}
],
"input_data_sets": [],
- "name": "0054-split-fields-id UNIT",
+ "name": "0003-field-splitter-id UNIT",
"description": "",
"persist_filename": "",
"trans_test_tweaks": [],
- "pipeline_filename": "./0054-split-fields-id.hpl",
+ "pipeline_filename": "./0003-field-splitter-id.hpl",
"test_type": "UNIT_TEST"
-}
\ No newline at end of file
+}
diff --git
a/integration-tests/transforms/metadata/unit-test/0054-split-fields-delimiter
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0003-field-splitter-keep-field
UNIT.json
similarity index 72%
rename from
integration-tests/transforms/metadata/unit-test/0054-split-fields-delimiter
UNIT.json
rename to
integration-tests/transforms/metadata/unit-test/0003-field-splitter-keep-field
UNIT.json
index e11735186d..be3b95292d 100644
---
a/integration-tests/transforms/metadata/unit-test/0054-split-fields-delimiter
UNIT.json
+++
b/integration-tests/transforms/metadata/unit-test/0003-field-splitter-keep-field
UNIT.json
@@ -10,6 +10,14 @@
"transform_field": "id",
"data_set_field": "id"
},
+ {
+ "transform_field": "value",
+ "data_set_field": "value"
+ },
+ {
+ "transform_field": "value2",
+ "data_set_field": "value2"
+ },
{
"transform_field": "sales1",
"data_set_field": "sales1"
@@ -25,24 +33,20 @@
{
"transform_field": "sales4",
"data_set_field": "sales4"
- },
- {
- "transform_field": "value2",
- "data_set_field": "value2"
}
],
"field_order": [
"id"
],
- "data_set_name": "golden-split-fields-delimiter",
+ "data_set_name": "golden-field-splitter-keep-field",
"transform_name": "Verify"
}
],
"input_data_sets": [],
- "name": "0054-split-fields-delimiter UNIT",
- "description": "",
+ "name": "0003-field-splitter-keep-field UNIT",
+ "description": "Keep the field to split and append new fields at the end",
"persist_filename": "",
"trans_test_tweaks": [],
- "pipeline_filename": "./0054-split-fields-delimiter.hpl",
+ "pipeline_filename": "./0003-field-splitter-keep-field.hpl",
"test_type": "UNIT_TEST"
-}
\ No newline at end of file
+}
diff --git
a/plugins/transforms/fieldsplitter/src/main/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitter.java
b/plugins/transforms/fieldsplitter/src/main/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitter.java
index ecc294af7c..346431a1f2 100644
---
a/plugins/transforms/fieldsplitter/src/main/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitter.java
+++
b/plugins/transforms/fieldsplitter/src/main/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitter.java
@@ -87,17 +87,24 @@ public class FieldSplitter extends
BaseTransform<FieldSplitterMeta, FieldSplitte
// reserve room
Object[] outputRow = RowDataUtil.allocateRowData(data.outputMeta.size());
- int nrExtraFields = meta.getFields().size() - 1;
-
- System.arraycopy(r, 0, outputRow, 0, data.fieldnr);
- System.arraycopy(
- r,
- data.fieldnr + 1,
- outputRow,
- data.fieldnr + 1 + nrExtraFields,
- data.previousMeta.size() - (data.fieldnr + 1));
+ final boolean keepSplitField = meta.isKeepSplitField();
+ if (keepSplitField) {
+ // Keep all original fields, append split results at the end
+ //
+ System.arraycopy(r, 0, outputRow, 0, data.previousMeta.size());
+ } else {
+ int nrExtraFields = meta.getFields().size() - 1;
+
+ System.arraycopy(r, 0, outputRow, 0, data.fieldnr);
+ System.arraycopy(
+ r,
+ data.fieldnr + 1,
+ outputRow,
+ data.fieldnr + 1 + nrExtraFields,
+ data.previousMeta.size() - (data.fieldnr + 1));
+ }
- // OK, now we have room in the middle to place the fields...
+ // OK, now we have room to place the fields...
//
// Named values info.id[0] not filled in!
@@ -118,6 +125,7 @@ public class FieldSplitter extends
BaseTransform<FieldSplitterMeta, FieldSplitte
Const.splitString(
valueToSplit, data.delimiter, data.enclosure, removeEnclosure,
data.escapeString);
int prev = 0;
+ final int splitOutputBaseIndex = keepSplitField ? data.previousMeta.size()
: data.fieldnr;
for (int i = 0; i < meta.getFields().size(); i++) {
FSField field = meta.getFields().get(i);
String id = Const.NVL(field.getId(), "");
@@ -156,8 +164,8 @@ public class FieldSplitter extends
BaseTransform<FieldSplitterMeta, FieldSplitte
Object value;
try {
- IValueMeta valueMeta = data.outputMeta.getValueMeta(data.fieldnr + i);
- IValueMeta conversionValueMeta =
data.conversionMeta.getValueMeta(data.fieldnr + i);
+ IValueMeta valueMeta =
data.outputMeta.getValueMeta(splitOutputBaseIndex + i);
+ IValueMeta conversionValueMeta =
data.conversionMeta.getValueMeta(splitOutputBaseIndex + i);
if (rawValue != null && valueMeta.isNull(rawValue)) {
rawValue = null;
@@ -178,7 +186,7 @@ public class FieldSplitter extends
BaseTransform<FieldSplitterMeta, FieldSplitte
meta.getSplitField() + "]!"),
e);
}
- outputRow[data.fieldnr + i] = value;
+ outputRow[splitOutputBaseIndex + i] = value;
}
return outputRow;
diff --git
a/plugins/transforms/fieldsplitter/src/main/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitterDialog.java
b/plugins/transforms/fieldsplitter/src/main/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitterDialog.java
index 07a0af4b67..e15af20e4d 100644
---
a/plugins/transforms/fieldsplitter/src/main/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitterDialog.java
+++
b/plugins/transforms/fieldsplitter/src/main/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitterDialog.java
@@ -23,8 +23,8 @@ import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.value.ValueMetaBase;
import org.apache.hop.core.row.value.ValueMetaFactory;
-import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
@@ -38,9 +38,12 @@ import org.apache.hop.ui.core.widget.TextVar;
import org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.CCombo;
+import org.eclipse.swt.events.SelectionAdapter;
+import org.eclipse.swt.events.SelectionEvent;
import org.eclipse.swt.graphics.Cursor;
import org.eclipse.swt.layout.FormAttachment;
import org.eclipse.swt.layout.FormData;
+import org.eclipse.swt.widgets.Button;
import org.eclipse.swt.widgets.Label;
import org.eclipse.swt.widgets.Shell;
import org.eclipse.swt.widgets.TableItem;
@@ -56,6 +59,8 @@ public class FieldSplitterDialog extends BaseTransformDialog {
private TextVar wEscapeString;
+ private Button wKeepSplitField;
+
private TableView wFields;
private final FieldSplitterMeta input;
@@ -160,12 +165,39 @@ public class FieldSplitterDialog extends
BaseTransformDialog {
fdEscapeString.right = new FormAttachment(100, 0);
wEscapeString.setLayoutData(fdEscapeString);
+ // Keep the field to split
+ Label wlKeepSplitField = new Label(shell, SWT.RIGHT);
+ wlKeepSplitField.setText(
+ BaseMessages.getString(PKG,
"FieldSplitterDialog.KeepSplitField.Label"));
+ PropsUi.setLook(wlKeepSplitField);
+ FormData fdlKeepSplitField = new FormData();
+ fdlKeepSplitField.top = new FormAttachment(wEscapeString, margin);
+ fdlKeepSplitField.left = new FormAttachment(0, 0);
+ fdlKeepSplitField.right = new FormAttachment(middle, -margin);
+ wlKeepSplitField.setLayoutData(fdlKeepSplitField);
+ wKeepSplitField = new Button(shell, SWT.CHECK);
+ wKeepSplitField.setToolTipText(
+ BaseMessages.getString(PKG,
"FieldSplitterDialog.KeepSplitField.Tooltip"));
+ PropsUi.setLook(wKeepSplitField);
+ FormData fdKeepSplitField = new FormData();
+ fdKeepSplitField.top = new FormAttachment(wlKeepSplitField, 0, SWT.CENTER);
+ fdKeepSplitField.left = new FormAttachment(middle, 0);
+ fdKeepSplitField.right = new FormAttachment(100, 0);
+ wKeepSplitField.setLayoutData(fdKeepSplitField);
+ wKeepSplitField.addSelectionListener(
+ new SelectionAdapter() {
+ @Override
+ public void widgetSelected(SelectionEvent e) {
+ input.setChanged();
+ }
+ });
+
Label wlFields = new Label(shell, SWT.RIGHT);
wlFields.setText(BaseMessages.getString(PKG,
"FieldSplitterDialog.Fields.Label"));
PropsUi.setLook(wlFields);
FormData fdlFields = new FormData();
fdlFields.left = new FormAttachment(0, 0);
- fdlFields.top = new FormAttachment(wEscapeString, margin);
+ fdlFields.top = new FormAttachment(wKeepSplitField, margin);
wlFields.setLayoutData(fdlFields);
final int fieldsRows = input.getFields().size();
@@ -224,7 +256,7 @@ public class FieldSplitterDialog extends
BaseTransformDialog {
new ColumnInfo(
BaseMessages.getString(PKG,
"FieldSplitterDialog.ColumnInfo.TrimType"),
ColumnInfo.COLUMN_TYPE_CCOMBO,
- ValueMetaString.trimTypeDesc,
+ ValueMetaBase.trimTypeDesc,
true),
};
wFields =
@@ -278,6 +310,7 @@ public class FieldSplitterDialog extends
BaseTransformDialog {
wDelimiter.setText(Const.NVL(input.getDelimiter(), ""));
wEnclosure.setText(Const.NVL(input.getEnclosure(), ""));
wEscapeString.setText(Const.NVL(input.getEscapeString(), ""));
+ wKeepSplitField.setSelection(input.isKeepSplitField());
for (int i = 0; i < input.getFields().size(); i++) {
FSField field = input.getFields().get(i);
@@ -320,6 +353,7 @@ public class FieldSplitterDialog extends
BaseTransformDialog {
input.setDelimiter(wDelimiter.getText());
input.setEnclosure(wEnclosure.getText());
input.setEscapeString(wEscapeString.getText());
+ input.setKeepSplitField(wKeepSplitField.getSelection());
input.getFields().clear();
for (TableItem ti : wFields.getNonEmptyItems()) {
diff --git
a/plugins/transforms/fieldsplitter/src/main/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitterMeta.java
b/plugins/transforms/fieldsplitter/src/main/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitterMeta.java
index 23d0997790..e7da47e15b 100644
---
a/plugins/transforms/fieldsplitter/src/main/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitterMeta.java
+++
b/plugins/transforms/fieldsplitter/src/main/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitterMeta.java
@@ -19,6 +19,8 @@ package org.apache.hop.pipeline.transforms.fieldsplitter;
import java.util.ArrayList;
import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.core.CheckResult;
import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.annotations.Transform;
@@ -35,6 +37,8 @@ import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
+@Getter
+@Setter
@Transform(
id = "FieldSplitter",
image = "fieldsplitter.svg",
@@ -74,6 +78,15 @@ public class FieldSplitterMeta extends
BaseTransformMeta<FieldSplitter, FieldSpl
injectionKeyDescription = "FieldSplitterDialog.EscapeString.Label")
private String escapeString;
+ /**
+ * When true, keep the field being split and append the new fields at the
end of the field list.
+ */
+ @HopMetadataProperty(
+ key = "keep_split_field",
+ injectionKey = "KEEP_SPLIT_FIELD",
+ injectionKeyDescription = "FieldSplitter.Injection.KEEP_SPLIT_FIELD")
+ private boolean keepSplitField;
+
@HopMetadataProperty(
groupKey = "fields",
key = "field",
@@ -93,6 +106,7 @@ public class FieldSplitterMeta extends
BaseTransformMeta<FieldSplitter, FieldSpl
this.delimiter = m.delimiter;
this.enclosure = m.enclosure;
this.escapeString = m.escapeString;
+ this.keepSplitField = m.keepSplitField;
m.fields.forEach(f -> this.fields.add(new FSField(f)));
}
@@ -106,6 +120,7 @@ public class FieldSplitterMeta extends
BaseTransformMeta<FieldSplitter, FieldSpl
splitField = "";
delimiter = ",";
enclosure = null;
+ keepSplitField = false;
}
@Override
@@ -127,6 +142,21 @@ public class FieldSplitterMeta extends
BaseTransformMeta<FieldSplitter, FieldSpl
PKG, "FieldSplitter.Log.CouldNotFindFieldToSplit",
getSplitField()));
}
+ if (keepSplitField) {
+ // Keep the original field and append the new fields at the end
+ //
+ for (FSField field : fields) {
+ try {
+ final IValueMeta v = field.createValueMeta();
+ v.setOrigin(name);
+ r.addValueMeta(v);
+ } catch (Exception e) {
+ throw new HopTransformException(e);
+ }
+ }
+ return;
+ }
+
// Add the new fields at the place of the index --> replace!
//
for (int i = 0; i < fields.size(); i++) {
@@ -218,6 +248,8 @@ public class FieldSplitterMeta extends
BaseTransformMeta<FieldSplitter, FieldSpl
}
}
+ @Getter
+ @Setter
public static class FSField {
@HopMetadataProperty(
key = "name",
@@ -333,297 +365,5 @@ public class FieldSplitterMeta extends
BaseTransformMeta<FieldSplitter, FieldSpl
valueMeta.setTrimType(trimType.getType());
return valueMeta;
}
-
- /**
- * Gets name
- *
- * @return value of name
- */
- public String getName() {
- return name;
- }
-
- /**
- * Sets name
- *
- * @param name value of name
- */
- public void setName(String name) {
- this.name = name;
- }
-
- /**
- * Gets id
- *
- * @return value of id
- */
- public String getId() {
- return id;
- }
-
- /**
- * Sets id
- *
- * @param id value of id
- */
- public void setId(String id) {
- this.id = id;
- }
-
- /**
- * Gets idRemoved
- *
- * @return value of idRemoved
- */
- public boolean isIdRemoved() {
- return idRemoved;
- }
-
- /**
- * Sets idRemoved
- *
- * @param idRemoved value of idRemoved
- */
- public void setIdRemoved(boolean idRemoved) {
- this.idRemoved = idRemoved;
- }
-
- /**
- * Gets type
- *
- * @return value of type
- */
- public String getType() {
- return type;
- }
-
- /**
- * Sets type
- *
- * @param type value of type
- */
- public void setType(String type) {
- this.type = type;
- }
-
- /**
- * Gets format
- *
- * @return value of format
- */
- public String getFormat() {
- return format;
- }
-
- /**
- * Sets format
- *
- * @param format value of format
- */
- public void setFormat(String format) {
- this.format = format;
- }
-
- /**
- * Gets group
- *
- * @return value of group
- */
- public String getGroup() {
- return group;
- }
-
- /**
- * Sets group
- *
- * @param group value of group
- */
- public void setGroup(String group) {
- this.group = group;
- }
-
- /**
- * Gets decimal
- *
- * @return value of decimal
- */
- public String getDecimal() {
- return decimal;
- }
-
- /**
- * Sets decimal
- *
- * @param decimal value of decimal
- */
- public void setDecimal(String decimal) {
- this.decimal = decimal;
- }
-
- /**
- * Gets currency
- *
- * @return value of currency
- */
- public String getCurrency() {
- return currency;
- }
-
- /**
- * Sets currency
- *
- * @param currency value of currency
- */
- public void setCurrency(String currency) {
- this.currency = currency;
- }
-
- /**
- * Gets length
- *
- * @return value of length
- */
- public int getLength() {
- return length;
- }
-
- /**
- * Sets length
- *
- * @param length value of length
- */
- public void setLength(int length) {
- this.length = length;
- }
-
- /**
- * Gets precision
- *
- * @return value of precision
- */
- public int getPrecision() {
- return precision;
- }
-
- /**
- * Sets precision
- *
- * @param precision value of precision
- */
- public void setPrecision(int precision) {
- this.precision = precision;
- }
-
- /**
- * Gets nullIf
- *
- * @return value of nullIf
- */
- public String getNullIf() {
- return nullIf;
- }
-
- /**
- * Sets nullIf
- *
- * @param nullIf value of nullIf
- */
- public void setNullIf(String nullIf) {
- this.nullIf = nullIf;
- }
-
- /**
- * Gets ifNull
- *
- * @return value of ifNull
- */
- public String getIfNull() {
- return ifNull;
- }
-
- /**
- * Sets ifNull
- *
- * @param ifNull value of ifNull
- */
- public void setIfNull(String ifNull) {
- this.ifNull = ifNull;
- }
-
- /**
- * Gets trimType
- *
- * @return value of trimType
- */
- public IValueMeta.TrimType getTrimType() {
- return trimType;
- }
-
- /**
- * Sets trimType
- *
- * @param trimType value of trimType
- */
- public void setTrimType(IValueMeta.TrimType trimType) {
- this.trimType = trimType;
- }
- }
-
- public String getSplitField() {
- return splitField;
- }
-
- public void setSplitField(final String splitField) {
- this.splitField = splitField;
- }
-
- public String getDelimiter() {
- return delimiter;
- }
-
- public void setDelimiter(final String delimiter) {
- this.delimiter = delimiter;
- }
-
- public String getEnclosure() {
- return enclosure;
- }
-
- public void setEnclosure(final String enclosure) {
- this.enclosure = enclosure;
- }
-
- /**
- * Gets escapeString
- *
- * @return value of escapeString
- */
- public String getEscapeString() {
- return escapeString;
- }
-
- /**
- * @param escapeString The escapeString to set
- */
- public void setEscapeString(String escapeString) {
- this.escapeString = escapeString;
- }
-
- /**
- * Gets fields
- *
- * @return value of fields
- */
- public List<FSField> getFields() {
- return fields;
- }
-
- /**
- * Sets fields
- *
- * @param fields value of fields
- */
- public void setFields(List<FSField> fields) {
- this.fields = fields;
}
}
diff --git
a/plugins/transforms/fieldsplitter/src/main/resources/org/apache/hop/pipeline/transforms/fieldsplitter/messages/messages_en_US.properties
b/plugins/transforms/fieldsplitter/src/main/resources/org/apache/hop/pipeline/transforms/fieldsplitter/messages/messages_en_US.properties
index 6779826fe1..c82f771737 100644
---
a/plugins/transforms/fieldsplitter/src/main/resources/org/apache/hop/pipeline/transforms/fieldsplitter/messages/messages_en_US.properties
+++
b/plugins/transforms/fieldsplitter/src/main/resources/org/apache/hop/pipeline/transforms/fieldsplitter/messages/messages_en_US.properties
@@ -26,6 +26,7 @@ FieldSplitter.Injection.FIELDS=All the resulting fields
FieldSplitter.Injection.FORMAT=The format (mask)
FieldSplitter.Injection.GROUPING=The grouping symbol
FieldSplitter.Injection.ID=The ID
+FieldSplitter.Injection.KEEP_SPLIT_FIELD=Keep the field to split and append
new fields at the end (Y/N)
FieldSplitter.Injection.LENGTH=Length
FieldSplitter.Injection.NAME=Field name
FieldSplitter.Injection.NULL_IF=Value to convert to null
@@ -63,6 +64,8 @@ FieldSplitterDialog.EscapeString.Tooltip=This will ignore a
delimiter if it's pr
FieldSplitterDialog.FailedToGetFields.DialogMessage=Error getting fields from
previous transforms
FieldSplitterDialog.FailedToGetFields.DialogTitle=Error getting fields
FieldSplitterDialog.Fields.Label=Fields
+FieldSplitterDialog.KeepSplitField.Label=Keep the field to split
+FieldSplitterDialog.KeepSplitField.Tooltip=Keep the original field and append
the new fields at the end of the field list
FieldSplitterDialog.Shell.Title=Split fields
FieldSplitterDialog.SplitField.Label=Field to split
FieldSplitterDialog.TransformName.Label=Transform name
diff --git
a/plugins/transforms/fieldsplitter/src/main/resources/org/apache/hop/pipeline/transforms/fieldsplitter/messages/messages_zh_CN.properties
b/plugins/transforms/fieldsplitter/src/main/resources/org/apache/hop/pipeline/transforms/fieldsplitter/messages/messages_zh_CN.properties
index 50b00947ee..0bc06bc57e 100644
---
a/plugins/transforms/fieldsplitter/src/main/resources/org/apache/hop/pipeline/transforms/fieldsplitter/messages/messages_zh_CN.properties
+++
b/plugins/transforms/fieldsplitter/src/main/resources/org/apache/hop/pipeline/transforms/fieldsplitter/messages/messages_zh_CN.properties
@@ -23,6 +23,7 @@ FieldSplitter.Injection.DECIMAL=\u5C0F\u6570\u70B9\u7B26\u53F7
FieldSplitter.Injection.DEFAULT=\u5982\u679C\u4E3A null
\u7684\u9ED8\u8BA4\u503C
FieldSplitter.Injection.DELIMITER=\u5206\u9694\u7B26
FieldSplitter.Injection.FIELD_TO_SPLIT=\u8981\u62C6\u5206\u7684\u5B57\u6BB5\u540D\u79F0
+FieldSplitter.Injection.KEEP_SPLIT_FIELD=\u4FDD\u7559\u88AB\u62C6\u5206\u5B57\u6BB5\u5E76\u5C06\u65B0\u5B57\u6BB5\u8FFD\u52A0\u5230\u672B\u5C3E
(Y/N)
FieldSplitter.Log.CouldNotFindFieldToSplit=Couldn''t find field to split [{0}]
in input stream\!
FieldSplitter.Log.ErrorConvertingSplitValue=Error converting value [{0}], when
splitting field [{1}
FieldSplitter.Log.LineNumber=linenr
@@ -54,6 +55,8 @@
FieldSplitterDialog.EscapeString.Tooltip=\u5F53\u5206\u9694\u7B26\u7D27\u8DDF\u5
FieldSplitterDialog.FailedToGetFields.DialogMessage=\u4ECE\u524D\u7F6E\u901A\u9053\u4E2D\u83B7\u53D6\u5B57\u6BB5\u65F6\u51FA\u9519
FieldSplitterDialog.FailedToGetFields.DialogTitle=\u9519\u8BEF\u83B7\u53D6\u5B57\u6BB5
FieldSplitterDialog.Fields.Label=\u5B57\u6BB5\:
+FieldSplitterDialog.KeepSplitField.Label=\u4FDD\u7559\u88AB\u62C6\u5206\u7684\u5B57\u6BB5
+FieldSplitterDialog.KeepSplitField.Tooltip=\u4FDD\u7559\u539F\u5B57\u6BB5\uFF0C\u5E76\u5C06\u62C6\u5206\u540E\u7684\u65B0\u5B57\u6BB5\u8FFD\u52A0\u5230\u5B57\u6BB5\u5217\u8868\u672B\u5C3E
FieldSplitterDialog.Shell.Title=\u62C6\u5206\u5B57\u6BB5
FieldSplitterDialog.SplitField.Label=\u9700\u8981\u62C6\u5206\u7684\u5B57\u6BB5:
FieldSplitterDialog.TransformName.Label=Transform \u540D\u79F0:
diff --git
a/plugins/transforms/fieldsplitter/src/test/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitterKeepSplitFieldTest.java
b/plugins/transforms/fieldsplitter/src/test/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitterKeepSplitFieldTest.java
new file mode 100644
index 0000000000..96a256d579
--- /dev/null
+++
b/plugins/transforms/fieldsplitter/src/test/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitterKeepSplitFieldTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.fieldsplitter;
+
+import static
org.apache.hop.pipeline.transforms.fieldsplitter.FieldSplitterMeta.FSField;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.PipelineTestingUtil;
+import org.apache.hop.pipeline.transform.ITransformData;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/** Unit test for {@link FieldSplitterMeta} */
+class FieldSplitterKeepSplitFieldTest {
+ private TransformMockHelper<FieldSplitterMeta, ITransformData> helper;
+
+ @RegisterExtension
+ static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+
+ @BeforeAll
+ static void initHop() throws Exception {
+ HopEnvironment.init();
+ }
+
+ @BeforeEach
+ void setUp() {
+ helper =
+ TransformMockUtil.getTransformMockHelper(
+ FieldSplitterMeta.class, "FieldSplitterKeepSplitFieldTest");
+ }
+
+ @AfterEach
+ void cleanUp() {
+ helper.cleanUp();
+ }
+
+ @Test
+ void getFieldsKeepsOriginalAndAppendsNewFields() throws Exception {
+ FieldSplitterMeta meta = createMeta(true);
+
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("id"));
+ rowMeta.addValueMeta(new ValueMetaString("movies"));
+ rowMeta.addValueMeta(new ValueMetaString("character"));
+
+ meta.getFields(rowMeta, "splitter", null, null, null, null);
+
+ assertEquals(5, rowMeta.size());
+ assertEquals("id", rowMeta.getValueMeta(0).getName());
+ assertEquals("movies", rowMeta.getValueMeta(1).getName());
+ assertEquals("character", rowMeta.getValueMeta(2).getName());
+ assertEquals("movie1", rowMeta.getValueMeta(3).getName());
+ assertEquals("movie2", rowMeta.getValueMeta(4).getName());
+ }
+
+ @Test
+ void getFieldsReplacesInPlaceByDefault() throws Exception {
+ FieldSplitterMeta meta = createMeta(false);
+
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("id"));
+ rowMeta.addValueMeta(new ValueMetaString("movies"));
+ rowMeta.addValueMeta(new ValueMetaString("character"));
+
+ meta.getFields(rowMeta, "splitter", null, null, null, null);
+
+ assertEquals(4, rowMeta.size());
+ assertEquals("id", rowMeta.getValueMeta(0).getName());
+ assertEquals("movie1", rowMeta.getValueMeta(1).getName());
+ assertEquals("movie2", rowMeta.getValueMeta(2).getName());
+ assertEquals("character", rowMeta.getValueMeta(3).getName());
+ }
+
+ @Test
+ void processRowKeepsOriginalAndAppendsValues() throws Exception {
+ FieldSplitterMeta meta = createMeta(true);
+ FieldSplitterData data = new FieldSplitterData();
+ FieldSplitter transform = createAndInitTransform(meta, data);
+
+ RowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaString("id"));
+ input.addValueMeta(new ValueMetaString("movies"));
+ input.addValueMeta(new ValueMetaString("character"));
+ transform.setInputRowMeta(input);
+
+ transform = spy(transform);
+ doReturn(new Object[] {"3", "A,B", "Captain
America"}).when(transform).getRow();
+
+ List<Object[]> actual = PipelineTestingUtil.execute(transform, 1, false);
+ PipelineTestingUtil.assertResult(
+ List.<Object[]>of(new Object[] {"3", "A,B", "Captain America", "A",
"B"}), actual);
+ }
+
+ private static FieldSplitterMeta createMeta(boolean keepSplitField) {
+ FieldSplitterMeta meta = new FieldSplitterMeta();
+ meta.setSplitField("movies");
+ meta.setDelimiter(",");
+ meta.setKeepSplitField(keepSplitField);
+
+ FSField f1 = new FSField();
+ f1.setName("movie1");
+ f1.setType("String");
+ FSField f2 = new FSField();
+ f2.setName("movie2");
+ f2.setType("String");
+ meta.getFields().addAll(List.of(f1, f2));
+ return meta;
+ }
+
+ private FieldSplitter createAndInitTransform(FieldSplitterMeta meta,
FieldSplitterData data) {
+ when(helper.transformMeta.getTransform()).thenReturn(meta);
+
+ FieldSplitter transform =
+ new FieldSplitter(
+ helper.transformMeta, meta, data, 0, helper.pipelineMeta,
helper.pipeline);
+ transform.init();
+ return transform;
+ }
+}
diff --git
a/plugins/transforms/fieldsplitter/src/test/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitterMetaTest.java
b/plugins/transforms/fieldsplitter/src/test/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitterMetaTest.java
index f6a85754b7..4bb1aa6827 100644
---
a/plugins/transforms/fieldsplitter/src/test/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitterMetaTest.java
+++
b/plugins/transforms/fieldsplitter/src/test/java/org/apache/hop/pipeline/transforms/fieldsplitter/FieldSplitterMetaTest.java
@@ -18,10 +18,12 @@
package org.apache.hop.pipeline.transforms.fieldsplitter;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import org.apache.hop.pipeline.transform.TransformSerializationTestUtil;
import org.junit.jupiter.api.Test;
+/** Unit test for {@link FieldSplitterMeta} */
class FieldSplitterMetaTest {
@Test
@@ -30,5 +32,6 @@ class FieldSplitterMetaTest {
TransformSerializationTestUtil.testSerialization(
"/field-splitter-transform.xml", FieldSplitterMeta.class);
assertEquals(4, meta.getFields().size());
+ assertFalse(meta.isKeepSplitField());
}
}
