This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 4858af6ca8 Fix#8266 Add Enclosure option to Split field to rows
(#8275)
4858af6ca8 is described below
commit 4858af6ca86250caeb9c5eb022b4d377427098c5
Author: Lance <[email protected]>
AuthorDate: Tue Sep 8 14:57:48 2026 +0800
Fix#8266 Add Enclosure option to Split field to rows (#8275)
* Add Enclosure option to Split field to rows
Signed-off-by: lance <[email protected]>
* Add Enclosure option to Split field to rows
Signed-off-by: lance <[email protected]>
---------
Signed-off-by: lance <[email protected]>
---
.../pipeline/transforms/splitfieldtorows.adoc | 5 +
integration-tests/deserialize/all-transforms.hpl | 1 +
.../0019-split-field-to-rows-enclosure.hpl | 150 ++++++++++++++++
.../golden-split-field-to-rows-enclosure.csv | 7 +
.../transforms/main-0019-split-field-to-rows.hwf | 3 +
.../golden-split-field-to-rows-enclosure.json | 32 ++++
.../0019-split-field-to-rows-enclosure UNIT.json | 37 ++++
.../splitfieldtorows/SplitFieldToRows.java | 63 ++++++-
.../splitfieldtorows/SplitFieldToRowsData.java | 2 +
.../splitfieldtorows/SplitFieldToRowsDialog.java | 35 +++-
.../splitfieldtorows/SplitFieldToRowsMeta.java | 22 +++
.../messages/messages_en_US.properties | 6 +-
.../messages/messages_zh_CN.properties | 4 +
.../splitfieldtorows/SplitFieldToRowsMetaTest.java | 1 +
.../splitfieldtorows/SplitFieldToRowsTest.java | 198 ++++++++++++++++++++-
15 files changed, 558 insertions(+), 8 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/splitfieldtorows.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/splitfieldtorows.adoc
index 140d42e20c..a4fb8a76ae 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/splitfieldtorows.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/splitfieldtorows.adoc
@@ -56,6 +56,7 @@ If the value type/format of the new row is not specified, the
new field will use
== Examples
* Delimiter Regular Expression to split on a comma, or a space, or a comma +
space (* n): ```[,\s]\s*```
+* Enclosure to keep commas inside quoted values: delimiter ```,``` and
enclosure ```"``` turns ```hi,"hello, world","hey"``` into three rows:
```hi```, ```hello, world```, ```hey```.
== Options
@@ -66,6 +67,10 @@ If the value type/format of the new row is not specified,
the new field will use
|Field to split|The field you want to split.
|Delimiter|The delimiter or separator used.
Special characters (e.g. CHAR ASCII HEX01) can be set with the format
$[value], e.g. $[01] or $[6F,FF,00,1F].
+|Enclosure|Optional enclosure string. Delimiters inside a pair of enclosure
characters are not treated as split points, and the enclosure characters are
removed from the output values.
+For example with a comma (,) delimiter: ```hi,"hello, world","hey"``` becomes
three rows using enclosure ```"```.
+Doubled enclosure characters inside a quoted value are treated as one literal
enclosure. Trailing empty values are preserved, matching the behaviour without
an enclosure. An unterminated enclosure is logged and the remainder of the
field is kept as one value.
+This option is ignored when *Delimiter is a Regular Expression* is enabled.
|Delimiter is a Regular Expression|The regular expression to match the
delimiter.
|New field name|The name of the new field.
|===
diff --git a/integration-tests/deserialize/all-transforms.hpl
b/integration-tests/deserialize/all-transforms.hpl
index dd920cc75a..b519a62a06 100644
--- a/integration-tests/deserialize/all-transforms.hpl
+++ b/integration-tests/deserialize/all-transforms.hpl
@@ -3388,6 +3388,7 @@ from
<schema_name/>
</partitioning>
<delimiter>;</delimiter>
+ <enclosure/>
<delimiter_is_regex>N</delimiter_is_regex>
<resetrownumber>Y</resetrownumber>
<rownum>N</rownum>
diff --git
a/integration-tests/transforms/0019-split-field-to-rows-enclosure.hpl
b/integration-tests/transforms/0019-split-field-to-rows-enclosure.hpl
new file mode 100644
index 0000000000..f29d68d425
--- /dev/null
+++ b/integration-tests/transforms/0019-split-field-to-rows-enclosure.hpl
@@ -0,0 +1,150 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0019-split-field-to-rows-enclosure</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/06 10:03:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/06 10:03:00.000</modified_date>
+ <key_for_session_key/>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Sample input</from>
+ <to>Split field to rows</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Split field to rows</from>
+ <to>Verify</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Sample input</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>csv</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>1</item>
+ <item>a,b,c</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>hi,"hello, world","hey"</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Split field to rows</name>
+ <type>SplitFieldToRows3</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <splitfield>csv</splitfield>
+ <delimiter>,</delimiter>
+ <enclosure>"</enclosure>
+ <newfield>value</newfield>
+ <rownum>Y</rownum>
+ <rownum_field>rowNr</rownum_field>
+ <resetrownumber>Y</resetrownumber>
+ <delimiter_is_regex>N</delimiter_is_regex>
+ <attributes/>
+ <GUI>
+ <xloc>352</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Verify</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>576</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-split-field-to-rows-enclosure.csv
b/integration-tests/transforms/datasets/golden-split-field-to-rows-enclosure.csv
new file mode 100644
index 0000000000..030f00c9df
--- /dev/null
+++
b/integration-tests/transforms/datasets/golden-split-field-to-rows-enclosure.csv
@@ -0,0 +1,7 @@
+id,value,rowNr
+1,a,1
+1,b,2
+1,c,3
+2,hi,1
+2,"hello, world",2
+2,hey,3
diff --git a/integration-tests/transforms/main-0019-split-field-to-rows.hwf
b/integration-tests/transforms/main-0019-split-field-to-rows.hwf
index 8409e288dd..14f48e2ee4 100644
--- a/integration-tests/transforms/main-0019-split-field-to-rows.hwf
+++ b/integration-tests/transforms/main-0019-split-field-to-rows.hwf
@@ -57,6 +57,9 @@ limitations under the License.
<test_name>
<name>0019-split-field-to-rows-basic UNIT</name>
</test_name>
+ <test_name>
+ <name>0019-split-field-to-rows-enclosure UNIT</name>
+ </test_name>
</test_names>
<parallel>N</parallel>
<xloc>272</xloc>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-split-field-to-rows-enclosure.json
b/integration-tests/transforms/metadata/dataset/golden-split-field-to-rows-enclosure.json
new file mode 100644
index 0000000000..25eed63e5e
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-split-field-to-rows-enclosure.json
@@ -0,0 +1,32 @@
+{
+ "base_filename": "golden-split-field-to-rows-enclosure.csv",
+ "name": "golden-split-field-to-rows-enclosure",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_format": "####0;-####0",
+ "field_name": "id"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "value"
+ },
+ {
+ "field_comment": "",
+ "field_length": 10,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_format": "####0;-####0",
+ "field_name": "rowNr"
+ }
+ ],
+ "folder_name": ""
+}
diff --git
a/integration-tests/transforms/metadata/unit-test/0019-split-field-to-rows-enclosure
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0019-split-field-to-rows-enclosure
UNIT.json
new file mode 100644
index 0000000000..c9842e7b7b
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0019-split-field-to-rows-enclosure
UNIT.json
@@ -0,0 +1,37 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": true,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "value",
+ "data_set_field": "value"
+ },
+ {
+ "transform_field": "rowNr",
+ "data_set_field": "rowNr"
+ }
+ ],
+ "field_order": [
+ "id",
+ "rowNr"
+ ],
+ "transform_name": "Verify",
+ "data_set_name": "golden-split-field-to-rows-enclosure"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0019-split-field-to-rows-enclosure UNIT",
+ "description": "",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename": "./0019-split-field-to-rows-enclosure.hpl",
+ "test_type": "UNIT_TEST"
+}
diff --git
a/plugins/transforms/splitfieldtorows/src/main/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRows.java
b/plugins/transforms/splitfieldtorows/src/main/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRows.java
index f7d5993f8c..f3346945be 100644
---
a/plugins/transforms/splitfieldtorows/src/main/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRows.java
+++
b/plugins/transforms/splitfieldtorows/src/main/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRows.java
@@ -17,6 +17,8 @@
package org.apache.hop.pipeline.transforms.splitfieldtorows;
+import java.util.ArrayList;
+import java.util.List;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import org.apache.hop.core.Const;
@@ -98,8 +100,8 @@ public class SplitFieldToRows extends
BaseTransform<SplitFieldToRowsMeta, SplitF
if (meta.isIncludeRowNumber() && meta.isResetRowNumber()) {
data.rownr = 1L;
}
- // use -1 for include all strings.
- String[] splitStrings = data.delimiterPattern.split(originalString, -1);
+
+ String[] splitStrings = splitSource(originalString);
for (String string : splitStrings) {
Object[] outputRow = RowDataUtil.createResizedCopy(rowData,
data.outputRowMeta.size());
outputRow[rowMeta.size()] = string;
@@ -114,6 +116,56 @@ public class SplitFieldToRows extends
BaseTransform<SplitFieldToRowsMeta, SplitF
return true;
}
+ /**
+ * Split using enclosure-aware parsing when an enclosure is set and the
delimiter is not a regular
+ * expression. Otherwise keep the historical Pattern.split behavior,
including trailing empty
+ * values.
+ */
+ private String[] splitSource(String originalString) {
+ if (Utils.isEmpty(data.enclosure) || meta.isIsDelimiterRegex()) {
+ // use -1 to include trailing empty strings
+ return data.delimiterPattern.split(originalString, -1);
+ }
+ return splitWithEnclosure(originalString);
+ }
+
+ /**
+ * Split on the delimiter, ignoring delimiters inside enclosures. Doubled
enclosures inside an
+ * enclosed value are kept as one literal enclosure. Trailing empty values
are preserved, matching
+ * the non-enclosure behaviour.
+ */
+ private String[] splitWithEnclosure(String source) {
+ String delimiter = data.delimiter;
+ String enclosure = data.enclosure;
+ List<String> values = new ArrayList<>();
+ StringBuilder value = new StringBuilder();
+ boolean inEnclosure = false;
+ int index = 0;
+ while (index < source.length()) {
+ if (source.startsWith(enclosure, index)) {
+ if (inEnclosure && source.startsWith(enclosure, index +
enclosure.length())) {
+ value.append(enclosure);
+ index += 2 * enclosure.length();
+ } else {
+ inEnclosure = !inEnclosure;
+ index += enclosure.length();
+ }
+ } else if (!inEnclosure && !delimiter.isEmpty() &&
source.startsWith(delimiter, index)) {
+ values.add(value.toString());
+ value.setLength(0);
+ index += delimiter.length();
+ } else {
+ value.append(source.charAt(index));
+ index++;
+ }
+ }
+ if (inEnclosure) {
+ logError(BaseMessages.getString(PKG,
"SplitFieldToRows.Log.UnterminatedEnclosure", source));
+ }
+ values.add(value.toString());
+ return values.toArray(new String[0]);
+ }
+
@Override
public synchronized boolean processRow() throws HopException {
@@ -144,11 +196,12 @@ public class SplitFieldToRows extends
BaseTransform<SplitFieldToRowsMeta, SplitF
data.rownr = 1L;
try {
- String delimiter = Const.nullToEmpty(meta.getDelimiter());
+ data.delimiter = resolve(Const.nullToEmpty(meta.getDelimiter()));
+ data.enclosure = resolve(Const.NVL(meta.getEnclosure(), ""));
if (meta.isIsDelimiterRegex()) {
- data.delimiterPattern = Pattern.compile(resolve(delimiter));
+ data.delimiterPattern = Pattern.compile(data.delimiter);
} else {
- data.delimiterPattern =
Pattern.compile(Pattern.quote(resolve(delimiter)));
+ data.delimiterPattern =
Pattern.compile(Pattern.quote(data.delimiter));
}
} catch (PatternSyntaxException pse) {
logError(pse.getMessage());
diff --git
a/plugins/transforms/splitfieldtorows/src/main/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsData.java
b/plugins/transforms/splitfieldtorows/src/main/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsData.java
index 5b95be1299..4d80a65159 100644
---
a/plugins/transforms/splitfieldtorows/src/main/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsData.java
+++
b/plugins/transforms/splitfieldtorows/src/main/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsData.java
@@ -30,6 +30,8 @@ public class SplitFieldToRowsData extends BaseTransformData
implements ITransfor
public IValueMeta splitMeta;
public long rownr;
public Pattern delimiterPattern;
+ public String delimiter;
+ public String enclosure;
public SplitFieldToRowsData() {
super();
diff --git
a/plugins/transforms/splitfieldtorows/src/main/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsDialog.java
b/plugins/transforms/splitfieldtorows/src/main/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsDialog.java
index 4a031fd62f..79d02858bd 100644
---
a/plugins/transforms/splitfieldtorows/src/main/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsDialog.java
+++
b/plugins/transforms/splitfieldtorows/src/main/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsDialog.java
@@ -51,6 +51,9 @@ public class SplitFieldToRowsDialog extends
BaseTransformDialog {
private TextVar wDelimiter;
+ private Label wlEnclosure;
+ private TextVar wEnclosure;
+
private TextVar wValName;
private Button wInclRownum;
@@ -139,6 +142,26 @@ public class SplitFieldToRowsDialog extends
BaseTransformDialog {
fdDelimiter.right = new FormAttachment(100, 0);
wDelimiter.setLayoutData(fdDelimiter);
+ // Enclosure line
+ wlEnclosure = new Label(shell, SWT.RIGHT);
+ wlEnclosure.setText(BaseMessages.getString(PKG,
"SplitFieldToRowsDialog.Enclosure.Label"));
+ PropsUi.setLook(wlEnclosure);
+ FormData fdlEnclosure = new FormData();
+ fdlEnclosure.left = new FormAttachment(0, 0);
+ fdlEnclosure.right = new FormAttachment(middle, -margin);
+ fdlEnclosure.top = new FormAttachment(wDelimiter, margin);
+ wlEnclosure.setLayoutData(fdlEnclosure);
+ wEnclosure = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT |
SWT.BORDER);
+ wEnclosure.setToolTipText(
+ BaseMessages.getString(PKG,
"SplitFieldToRowsDialog.Enclosure.Tooltip"));
+ PropsUi.setLook(wEnclosure);
+ wEnclosure.addModifyListener(lsMod);
+ FormData fdEnclosure = new FormData();
+ fdEnclosure.left = new FormAttachment(middle, 0);
+ fdEnclosure.top = new FormAttachment(wDelimiter, margin);
+ fdEnclosure.right = new FormAttachment(100, 0);
+ wEnclosure.setLayoutData(fdEnclosure);
+
// Add File to the result files name
Label wlDelimiterIsRegex = new Label(shell, SWT.RIGHT);
wlDelimiterIsRegex.setText(
@@ -146,7 +169,7 @@ public class SplitFieldToRowsDialog extends
BaseTransformDialog {
PropsUi.setLook(wlDelimiterIsRegex);
FormData fdlDelimiterIsRegex = new FormData();
fdlDelimiterIsRegex.left = new FormAttachment(0, 0);
- fdlDelimiterIsRegex.top = new FormAttachment(wDelimiter, margin);
+ fdlDelimiterIsRegex.top = new FormAttachment(wEnclosure, margin);
fdlDelimiterIsRegex.right = new FormAttachment(middle, -margin);
wlDelimiterIsRegex.setLayoutData(fdlDelimiterIsRegex);
wDelimiterIsRegex = new Button(shell, SWT.CHECK);
@@ -162,6 +185,7 @@ public class SplitFieldToRowsDialog extends
BaseTransformDialog {
new SelectionAdapter() {
@Override
public void widgetSelected(SelectionEvent arg0) {
+ setEnclosureEnabled();
input.setChanged();
}
};
@@ -269,6 +293,7 @@ public class SplitFieldToRowsDialog extends
BaseTransformDialog {
getData();
setIncludeRownum();
+ setEnclosureEnabled();
input.setChanged(changed);
focusTransformName();
BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel());
@@ -282,9 +307,16 @@ public class SplitFieldToRowsDialog extends
BaseTransformDialog {
wResetRownum.setEnabled(wInclRownum.getSelection());
}
+ public void setEnclosureEnabled() {
+ boolean enabled = !wDelimiterIsRegex.getSelection();
+ wlEnclosure.setEnabled(enabled);
+ wEnclosure.setEnabled(enabled);
+ }
+
public void getData() {
wSplitField.setText(Const.NVL(input.getSplitField(), ""));
wDelimiter.setText(Const.NVL(input.getDelimiter(), ""));
+ wEnclosure.setText(Const.NVL(input.getEnclosure(), ""));
wValName.setText(Const.NVL(input.getNewFieldname(), ""));
wInclRownum.setSelection(input.isIncludeRowNumber());
wDelimiterIsRegex.setSelection(input.isIsDelimiterRegex());
@@ -308,6 +340,7 @@ public class SplitFieldToRowsDialog extends
BaseTransformDialog {
transformName = wTransformName.getText(); // return value
input.setSplitField(wSplitField.getText());
input.setDelimiter(wDelimiter.getText());
+ input.setEnclosure(wEnclosure.getText());
input.setNewFieldname(wValName.getText());
input.setIncludeRowNumber(wInclRownum.getSelection());
input.setRowNumberField(wInclRownumField.getText());
diff --git
a/plugins/transforms/splitfieldtorows/src/main/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsMeta.java
b/plugins/transforms/splitfieldtorows/src/main/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsMeta.java
index e89a5f5f33..232c9c441c 100644
---
a/plugins/transforms/splitfieldtorows/src/main/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsMeta.java
+++
b/plugins/transforms/splitfieldtorows/src/main/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsMeta.java
@@ -61,6 +61,13 @@ public class SplitFieldToRowsMeta
injectionKeyDescription =
"SplitFieldsToRow.Injection.Delimiter.Description")
private String delimiter;
+ /** Ignore delimiter inside pairs of the enclosure string */
+ @HopMetadataProperty(
+ key = "enclosure",
+ injectionKey = "ENCLOSURE",
+ injectionKeyDescription =
"SplitFieldsToRow.Injection.Enclosure.Description")
+ private String enclosure;
+
/** New name of the split field */
@HopMetadataProperty(
key = "newfield",
@@ -122,6 +129,20 @@ public class SplitFieldToRowsMeta
this.delimiter = delimiter;
}
+ /**
+ * @return Returns the enclosure.
+ */
+ public String getEnclosure() {
+ return enclosure;
+ }
+
+ /**
+ * @param enclosure The enclosure to set.
+ */
+ public void setEnclosure(String enclosure) {
+ this.enclosure = enclosure;
+ }
+
/**
* @return Returns the splitField.
*/
@@ -140,6 +161,7 @@ public class SplitFieldToRowsMeta
public void setDefault() {
splitField = "";
delimiter = ";";
+ enclosure = "";
newFieldname = "";
includeRowNumber = false;
isDelimiterRegex = false;
diff --git
a/plugins/transforms/splitfieldtorows/src/main/resources/org/apache/hop/pipeline/transforms/splitfieldtorows/messages/messages_en_US.properties
b/plugins/transforms/splitfieldtorows/src/main/resources/org/apache/hop/pipeline/transforms/splitfieldtorows/messages/messages_en_US.properties
index 90f48c5a10..170f97675f 100644
---
a/plugins/transforms/splitfieldtorows/src/main/resources/org/apache/hop/pipeline/transforms/splitfieldtorows/messages/messages_en_US.properties
+++
b/plugins/transforms/splitfieldtorows/src/main/resources/org/apache/hop/pipeline/transforms/splitfieldtorows/messages/messages_en_US.properties
@@ -17,6 +17,7 @@
SplitFieldsToRow.Injection.Delimiter.Description=Field''s delimiter character
SplitFieldsToRow.Injection.DelimiterIsRegexp.Description=Delimiter is a
Regular Expression
+SplitFieldsToRow.Injection.Enclosure.Description=Enclosure string. Delimiters
inside enclosure pairs are not treated as split points.
SplitFieldsToRow.Injection.FieldToSplit.Description=Field to split
SplitFieldsToRow.Injection.IncludeRowNum.Description=Include rownum field in
output?(Y/N)
SplitFieldsToRow.Injection.NewFieldname.Description=New field name
@@ -28,10 +29,13 @@ SplitFieldToRows.Log.CouldNotFindFieldToSplit=Couldn''t
find field to split [{0}
SplitFieldToRows.Log.LineNumber=linenr
SplitFieldToRows.Log.NewFieldNameIsNull=New field name not specified\!
SplitFieldToRows.Log.SplitFieldNotValid=Can only split string fields. [{0}]
is not a string\!
+SplitFieldToRows.Log.UnterminatedEnclosure=Unterminated enclosure in value
[{0}], the remainder was treated as one value.
SplitFieldToRows.Name=Split field to rows
SplitFieldToRowsDialog.Delimiter.Label=Delimiter
SplitFieldToRowsDialog.DelimiterIsRegex.Label=Delimiter is a Regular Expression
SplitFieldToRowsDialog.DelimiterIsRegex.Tooltip=Check this if the delimiter is
to be interpreted as a regular expression.\nRegex features like lookahead and
lookbehind often prove very useful in string split contexts.
+SplitFieldToRowsDialog.Enclosure.Label=Enclosure
+SplitFieldToRowsDialog.Enclosure.Tooltip=Ignore delimiters within pairs of
enclosures (leave blank if none).\nEnclosure characters are removed from the
output values.\nThis option is ignored when the delimiter is a regular
expression.
SplitFieldToRowsDialog.InclRownum.Label=Include rownum in output
SplitFieldToRowsDialog.InclRownum.Tooltip=Check this to add a field (Integer)
containing the row number.
SplitFieldToRowsDialog.InclRownumField.Label=Rownum fieldname
@@ -52,4 +56,4 @@ SplitFieldToRowsMeta.CheckResult.RowNumberFieldMissing=Row
number field is missi
SplitFieldToRowsMeta.CheckResult.RowNumberFieldOk=Row number field is
specified.
SplitFieldToRowsMeta.CheckResult.TransformReceivingFields=Transform is
connected to previous one, receiving {0} fields
SplitFieldToRowsMeta.CheckResult.TransformReceivingInfoFromOtherTransform=Transform
is receiving info from other transforms.
-SplitFieldToRowsMeta.keyword=split,explode,delimiter,tokenize,separate
+SplitFieldToRowsMeta.keyword=split,explode,delimiter,tokenize,separate,enclosure
diff --git
a/plugins/transforms/splitfieldtorows/src/main/resources/org/apache/hop/pipeline/transforms/splitfieldtorows/messages/messages_zh_CN.properties
b/plugins/transforms/splitfieldtorows/src/main/resources/org/apache/hop/pipeline/transforms/splitfieldtorows/messages/messages_zh_CN.properties
index 9a15defb9f..93778c51b1 100644
---
a/plugins/transforms/splitfieldtorows/src/main/resources/org/apache/hop/pipeline/transforms/splitfieldtorows/messages/messages_zh_CN.properties
+++
b/plugins/transforms/splitfieldtorows/src/main/resources/org/apache/hop/pipeline/transforms/splitfieldtorows/messages/messages_zh_CN.properties
@@ -19,6 +19,7 @@
SplitFieldsToRow.Injection.Delimiter.Description=\u5B57\u6BB5\u7684\u5206\u9694\u7B26
SplitFieldsToRow.Injection.DelimiterIsRegexp.Description=\u5206\u9694\u7B26\u662F\u6B63\u5219\u8868\u8FBE\u5F0F
+SplitFieldsToRow.Injection.Enclosure.Description=\u5305\u56F4\u7B26\u3002\u5305\u56F4\u7B26\u5BF9\u4E4B\u5185\u7684\u5206\u9694\u7B26\u4E0D\u4F5C\u62C6\u5206\u70B9\u3002
SplitFieldsToRow.Injection.FieldToSplit.Description=\u8981\u62C6\u5206\u7684\u5B57\u6BB5
SplitFieldsToRow.Injection.IncludeRowNum.Description=\u5728\u8F93\u51FA\u4E2D\u5305\u542B\u884C\u53F7\u5B57\u6BB5?(Y/N)
SplitFieldsToRow.Injection.NewFieldname.Description=\u65B0\u5B57\u6BB5\u540D
@@ -28,10 +29,13 @@
SplitFieldToRows.Log.CouldNotFindFieldToSplit=\u4ECE\u524D\u7F6E\u901A\u9053\u4E
SplitFieldToRows.Log.LineNumber=\u884C\u53F7
SplitFieldToRows.Log.NewFieldNameIsNull=\u672A\u6307\u5B9A\u65B0\u5B57\u6BB5\u540D\u79F0\!
SplitFieldToRows.Log.SplitFieldNotValid=\u53EA\u80FD\u5206\u9694\u5B57\u7B26\u7C7B\u578B\u5B57\u6BB5,
[{0}] \u4E0D\u662F\u5B57\u7B26\u4E3A\u7C7B\u578B\!
+SplitFieldToRows.Log.UnterminatedEnclosure=\u503C [{0}]
\u4E2D\u7684\u5305\u56F4\u7B26\u672A\u95ED\u5408\uFF0C\u5269\u4F59\u90E8\u5206\u5DF2\u4F5C\u4E3A\u5355\u4E2A\u503C\u5904\u7406\u3002
SplitFieldToRows.Name=\u5217\u62C6\u5206\u4E3A\u591A\u884C
SplitFieldToRowsDialog.Delimiter.Label=\u5206\u5272\u7B26:
SplitFieldToRowsDialog.DelimiterIsRegex.Label=\u5206\u9694\u7B26\u662F\u4E00\u4E2A\u6B63\u5219\u8868\u8FBE\u5F0F\:
SplitFieldToRowsDialog.DelimiterIsRegex.Tooltip=\u542F\u7528\u8BE5\u9879\u5219\u5C06\u5206\u9694\u7B26\u89C6\u4E3A\u6B63\u5219\u8868\u8FBE\u5F0F\u8FDB\u884C\u5904\u7406.\n\u50CF\u524D\u77BB\u548C\u540E\u89C6\u8FD9\u6837\u7684\u6B63\u5219\u8868\u8FBE\u5F0F\u529F\u80FD,
\u901A\u5E38\u5728\u5B57\u7B26\u4E32\u62C6\u5206\u4E0A\u4E0B\u6587\u4E2D\u975E\u5E38\u6709\u7528
+SplitFieldToRowsDialog.Enclosure.Label=\u5305\u56F4\u7B26:
+SplitFieldToRowsDialog.Enclosure.Tooltip=\u5FFD\u7565\u5305\u56F4\u7B26\u5BF9\u4E4B\u5185\u7684\u5206\u9694\u7B26\uFF08\u65E0\u5219\u7559\u7A7A\uFF09\u3002\n\u8F93\u51FA\u503C\u4F1A\u53BB\u6389\u5305\u56F4\u7B26\u3002\n\u5F53\u5206\u9694\u7B26\u662F\u6B63\u5219\u8868\u8FBE\u5F0F\u65F6\u6B64\u9009\u9879\u4E0D\u751F\u6548\u3002
SplitFieldToRowsDialog.InclRownum.Label=\u8F93\u51FA\u4E2D\u5305\u62EC\u884C\u53F7:
SplitFieldToRowsDialog.InclRownum.Tooltip=\u5728\u8F93\u51FA\u6D41\u4E2D\u589E\u52A0\u4E00\u4E2A\u884C\u53F7\u5B57\u6BB5
SplitFieldToRowsDialog.InclRownumField.Label=\u884C\u53F7\u5B57\u6BB5:
diff --git
a/plugins/transforms/splitfieldtorows/src/test/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsMetaTest.java
b/plugins/transforms/splitfieldtorows/src/test/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsMetaTest.java
index 7e373df4e3..4f14449b1e 100644
---
a/plugins/transforms/splitfieldtorows/src/test/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsMetaTest.java
+++
b/plugins/transforms/splitfieldtorows/src/test/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsMetaTest.java
@@ -45,6 +45,7 @@ class SplitFieldToRowsMetaTest {
Arrays.asList(
"splitField",
"delimiter",
+ "enclosure",
"newFieldname",
"includeRowNumber",
"rowNumberField",
diff --git
a/plugins/transforms/splitfieldtorows/src/test/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsTest.java
b/plugins/transforms/splitfieldtorows/src/test/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsTest.java
index e77e651cc9..bfe70166f6 100644
---
a/plugins/transforms/splitfieldtorows/src/test/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsTest.java
+++
b/plugins/transforms/splitfieldtorows/src/test/java/org/apache/hop/pipeline/transforms/splitfieldtorows/SplitFieldToRowsTest.java
@@ -18,19 +18,44 @@
package org.apache.hop.pipeline.transforms.splitfieldtorows;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.contains;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import org.apache.hop.core.BlockingRowSet;
+import org.apache.hop.core.HopEnvironment;
import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
class SplitFieldToRowsTest {
+ @RegisterExtension
+ static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+
private TransformMockHelper<SplitFieldToRowsMeta, SplitFieldToRowsData>
transformMockHelper;
+ @BeforeAll
+ static void initHop() throws Exception {
+ HopEnvironment.init();
+ }
+
@BeforeEach
void setup() {
transformMockHelper =
@@ -47,7 +72,7 @@ class SplitFieldToRowsTest {
}
@Test
- void interpretsNullDelimiterAsEmpty() throws Exception {
+ void interpretsNullDelimiterAsEmpty() {
SplitFieldToRows transform =
new SplitFieldToRows(
transformMockHelper.transformMeta,
@@ -68,4 +93,175 @@ class SplitFieldToRowsTest {
// empty string should be quoted --> \Q\E
assertEquals("\\Q\\E", transform.getData().delimiterPattern.pattern());
}
+
+ @Test
+ void splitsWithoutEnclosure() throws Exception {
+ List<Object[]> rows = executeSplit("a,b,c", ",", null, false);
+ assertEquals(List.of("a", "b", "c"), values(rows));
+ }
+
+ @Test
+ void splitsQuotedValuesWithEnclosure() throws Exception {
+ List<Object[]> rows = executeSplit("hi,\"hello, world\",\"hey\"", ",",
"\"", false);
+ assertEquals(List.of("hi", "hello, world", "hey"), values(rows));
+ }
+
+ @Test
+ void removesEnclosureFromSimpleQuotedValues() throws Exception {
+ List<Object[]> rows = executeSplit("\"a\",\"b\",\"c\"", ",", "\"", false);
+ assertEquals(List.of("a", "b", "c"), values(rows));
+ }
+
+ @Test
+ void keepsDelimiterInsideEnclosure() throws Exception {
+ List<Object[]> rows = executeSplit("x,\"y,z\",w", ",", "\"", false);
+ assertEquals(List.of("x", "y,z", "w"), values(rows));
+ }
+
+ @Test
+ void splitsQuotedValuesIntoFourRowsWithoutEnclosure() throws Exception {
+ List<Object[]> rows = executeSplit("hi,\"hello, world\",\"hey\"", ",",
null, false);
+ assertEquals(List.of("hi", "\"hello", " world\"", "\"hey\""),
values(rows));
+ }
+
+ @Test
+ void splitsWithRegexDelimiter() throws Exception {
+ List<Object[]> rows = executeSplit("a, b,c", ",\\s*", null, true);
+ assertEquals(List.of("a", "b", "c"), values(rows));
+ }
+
+ @Test
+ void ignoresEnclosureWhenDelimiterIsRegex() throws Exception {
+ List<Object[]> rows = executeSplit("hi,\"hello, world\",\"hey\"", ",",
"\"", true);
+ assertEquals(List.of("hi", "\"hello", " world\"", "\"hey\""),
values(rows));
+ }
+
+ @Test
+ void preservesTrailingEmptyValuesWithoutEnclosure() throws Exception {
+ List<Object[]> rows = executeSplit("a,b,,", ",", null, false);
+ assertEquals(List.of("a", "b", "", ""), values(rows));
+ }
+
+ @Test
+ void preservesTrailingEmptyValuesWithEnclosure() throws Exception {
+ List<Object[]> rows = executeSplit("a,b,,", ",", "\"", false);
+ assertEquals(List.of("a", "b", "", ""), values(rows));
+ }
+
+ @Test
+ void splitsLoneDelimiterIntoTwoEmptyValuesWithEnclosure() throws Exception {
+ List<Object[]> rows = executeSplit(",", ",", "\"", false);
+ assertEquals(List.of("", ""), values(rows));
+ }
+
+ @Test
+ void keepsRemainderAndLogsUnterminatedEnclosure() throws Exception {
+ List<Object[]> rows = executeSplit("a,\"b", ",", "\"", false);
+ assertEquals(List.of("a", "b"), values(rows));
+ verify(transformMockHelper.iLogChannel).logError(contains("Unterminated
enclosure"));
+ }
+
+ @Test
+ void doesNotDropRowOnStrayEnclosure() throws Exception {
+ List<Object[]> rows = executeSplit("a\"b,c", ",", "\"", false);
+ assertEquals(List.of("ab,c"), values(rows));
+ verify(transformMockHelper.iLogChannel).logError(contains("Unterminated
enclosure"));
+ }
+
+ @Test
+ void unescapesDoubledEnclosureInsideQuotedValue() throws Exception {
+ List<Object[]> rows = executeSplit("\"a,b\",\"c\"\"d\"", ",", "\"", false);
+ assertEquals(List.of("a,b", "c\"d"), values(rows));
+ }
+
+ @Test
+ void resolvesEnclosureFromVariable() throws Exception {
+ List<Object[]> rows =
+ executeSplit(
+ createMeta(",", "${ENCL}", false), "hi,\"hello, world\",\"hey\"",
Map.of("ENCL", "\""));
+ assertEquals(List.of("hi", "hello, world", "hey"), values(rows));
+ }
+
+ @Test
+ void includesResetRowNumbers() throws Exception {
+ SplitFieldToRowsMeta meta = createMeta(",", "\"", false);
+ meta.setIncludeRowNumber(true);
+ meta.setRowNumberField("rowNr");
+ meta.setResetRowNumber(true);
+
+ List<Object[]> rows = executeSplit(meta, "hi,\"hello, world\",\"hey\"");
+ assertEquals(3, rows.size());
+ assertEquals("hi", rows.get(0)[1]);
+ assertEquals(1L, rows.get(0)[2]);
+ assertEquals("hello, world", rows.get(1)[1]);
+ assertEquals(2L, rows.get(1)[2]);
+ assertEquals("hey", rows.get(2)[1]);
+ assertEquals(3L, rows.get(2)[2]);
+ }
+
+ private List<Object[]> executeSplit(
+ String value, String delimiter, String enclosure, boolean
delimiterIsRegex) throws Exception {
+ return executeSplit(createMeta(delimiter, enclosure, delimiterIsRegex),
value);
+ }
+
+ private List<Object[]> executeSplit(SplitFieldToRowsMeta meta, String value)
throws Exception {
+ return executeSplit(meta, value, Map.of());
+ }
+
+ private List<Object[]> executeSplit(
+ SplitFieldToRowsMeta meta, String value, Map<String, String> variables)
throws Exception {
+ SplitFieldToRowsData data = new SplitFieldToRowsData();
+ when(transformMockHelper.transformMeta.getTransform()).thenReturn(meta);
+
+ SplitFieldToRows transform =
+ new SplitFieldToRows(
+ transformMockHelper.transformMeta,
+ meta,
+ data,
+ 0,
+ transformMockHelper.pipelineMeta,
+ transformMockHelper.pipeline);
+ variables.forEach(transform::setVariable);
+ transform.init();
+
+ RowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaString("csv"));
+ transform.setInputRowMeta(input);
+
+ BlockingRowSet output = new BlockingRowSet(20);
+ transform.setOutputRowSets(Collections.singletonList(output));
+
+ SplitFieldToRows spyTransform = spy(transform);
+ doReturn(new Object[] {value}).doReturn(null).when(spyTransform).getRow();
+
+ assertTrue(spyTransform.processRow());
+ assertFalse(spyTransform.processRow());
+
+ List<Object[]> result = new ArrayList<>();
+ Object[] row;
+ while ((row = output.getRowImmediate()) != null) {
+ result.add(row);
+ }
+ return result;
+ }
+
+ private static SplitFieldToRowsMeta createMeta(
+ String delimiter, String enclosure, boolean delimiterIsRegex) {
+ SplitFieldToRowsMeta meta = new SplitFieldToRowsMeta();
+ meta.setSplitField("csv");
+ meta.setDelimiter(delimiter);
+ meta.setEnclosure(enclosure);
+ meta.setNewFieldname("value");
+ meta.setIsDelimiterRegex(delimiterIsRegex);
+ meta.setIncludeRowNumber(false);
+ return meta;
+ }
+
+ private static List<String> values(List<Object[]> rows) {
+ List<String> values = new ArrayList<>();
+ for (Object[] row : rows) {
+ values.add((String) row[1]);
+ }
+ return values;
+ }
}
