This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new d3f07ffbc9 Fix Align CSV Input Verify with filename-from-field and
static filename modes (#8502)
d3f07ffbc9 is described below
commit d3f07ffbc995f0863f345ebf5c269453de64c7cc
Author: Lance <[email protected]>
AuthorDate: Tue Sep 22 16:05:53 2026 +0800
Fix Align CSV Input Verify with filename-from-field and static filename
modes (#8502)
Signed-off-by: leehaut <[email protected]>
---
.../pipeline/transforms/csvinput/CsvInputMeta.java | 115 ++++++++++++++----
.../csvinput/messages/messages_en_US.properties | 5 +
.../csvinput/messages/messages_zh_CN.properties | 5 +
.../transforms/csvinput/CsvInputMetaTest.java | 130 +++++++++++++++++++++
4 files changed, 232 insertions(+), 23 deletions(-)
diff --git
a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputMeta.java
b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputMeta.java
index 92610114ef..fbf0878a9d 100644
---
a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputMeta.java
+++
b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputMeta.java
@@ -389,41 +389,110 @@ public class CsvInputMeta extends
BaseTransformMeta<CsvInput, CsvInputData>
IRowMeta info,
IVariables variables,
IHopMetadataProvider metadataProvider) {
- CheckResult cr;
+ // Same split as the dialog and CsvInput.init(): a previous hop means the
filename comes from a
+ // field, otherwise a static filename is used.
+ if (input != null && input.length > 0) {
+ checkFilenameFromField(remarks, transformMeta, prev, variables);
+ } else {
+ checkStaticFilename(remarks, transformMeta, prev, variables);
+ }
+ }
+
+ private void checkFilenameFromField(
+ List<ICheckResult> remarks,
+ TransformMeta transformMeta,
+ IRowMeta prev,
+ IVariables variables) {
+ remarks.add(
+ new CheckResult(
+ ICheckResult.TYPE_RESULT_OK,
+ BaseMessages.getString(PKG,
"CsvInputMeta.CheckResult.TransformRecevingData2"),
+ transformMeta));
+
if (prev == null || prev.isEmpty()) {
- cr =
+ remarks.add(
+ new CheckResult(
+ ICheckResult.TYPE_RESULT_ERROR,
+ BaseMessages.getString(PKG,
"CsvInputMeta.CheckResult.NotReceivingFields"),
+ transformMeta));
+ return;
+ }
+
+ remarks.add(
+ new CheckResult(
+ ICheckResult.TYPE_RESULT_OK,
+ BaseMessages.getString(
+ PKG, "CsvInputMeta.CheckResult.TransformRecevingData",
prev.size() + ""),
+ transformMeta));
+ remarks.add(checkFilenameField(transformMeta, prev, variables));
+ }
+
+ private CheckResult checkFilenameField(
+ TransformMeta transformMeta, IRowMeta prev, IVariables variables) {
+ if (Utils.isEmpty(filenameField)) {
+ return new CheckResult(
+ ICheckResult.TYPE_RESULT_ERROR,
+ BaseMessages.getString(PKG,
"CsvInputMeta.CheckResult.FilenameFieldMissing"),
+ transformMeta);
+ }
+
+ String resolvedField = variables != null ?
variables.resolve(filenameField) : filenameField;
+ if (prev.indexOfValue(resolvedField) < 0) {
+ return new CheckResult(
+ ICheckResult.TYPE_RESULT_ERROR,
+ BaseMessages.getString(
+ PKG, "CsvInputMeta.CheckResult.FilenameFieldNotFound",
resolvedField),
+ transformMeta);
+ }
+
+ return new CheckResult(
+ ICheckResult.TYPE_RESULT_OK,
+ BaseMessages.getString(PKG,
"CsvInputMeta.CheckResult.FilenameFieldOk", resolvedField),
+ transformMeta);
+ }
+
+ private void checkStaticFilename(
+ List<ICheckResult> remarks,
+ TransformMeta transformMeta,
+ IRowMeta prev,
+ IVariables variables) {
+ if (prev == null || prev.isEmpty()) {
+ remarks.add(
new CheckResult(
ICheckResult.TYPE_RESULT_OK,
BaseMessages.getString(PKG,
"CsvInputMeta.CheckResult.NotReceivingFields"),
- transformMeta);
- remarks.add(cr);
+ transformMeta));
} else {
- cr =
+ remarks.add(
new CheckResult(
ICheckResult.TYPE_RESULT_ERROR,
BaseMessages.getString(
PKG, "CsvInputMeta.CheckResult.TransformRecevingData",
prev.size() + ""),
- transformMeta);
- remarks.add(cr);
+ transformMeta));
}
- // See if we have input streams leading to this transform!
- if (input.length > 0) {
- cr =
- new CheckResult(
- ICheckResult.TYPE_RESULT_ERROR,
- BaseMessages.getString(PKG,
"CsvInputMeta.CheckResult.TransformRecevingData2"),
- transformMeta);
- remarks.add(cr);
- } else {
- cr =
- new CheckResult(
- ICheckResult.TYPE_RESULT_OK,
- BaseMessages.getString(
- PKG,
"CsvInputMeta.CheckResult.NoInputReceivedFromOtherTransforms"),
- transformMeta);
- remarks.add(cr);
+ remarks.add(
+ new CheckResult(
+ ICheckResult.TYPE_RESULT_OK,
+ BaseMessages.getString(
+ PKG,
"CsvInputMeta.CheckResult.NoInputReceivedFromOtherTransforms"),
+ transformMeta));
+ remarks.add(checkStaticFilenameSpecified(transformMeta, variables));
+ }
+
+ private CheckResult checkStaticFilenameSpecified(
+ TransformMeta transformMeta, IVariables variables) {
+ String resolvedFilename = variables != null ? variables.resolve(filename)
: filename;
+ if (Utils.isEmpty(resolvedFilename)) {
+ return new CheckResult(
+ ICheckResult.TYPE_RESULT_ERROR,
+ BaseMessages.getString(PKG,
"CsvInputMeta.CheckResult.FilenameMissing"),
+ transformMeta);
}
+ return new CheckResult(
+ ICheckResult.TYPE_RESULT_OK,
+ BaseMessages.getString(PKG, "CsvInputMeta.CheckResult.FilenameOk"),
+ transformMeta);
}
@Override
diff --git
a/plugins/transforms/textfile/src/main/resources/org/apache/hop/pipeline/transforms/csvinput/messages/messages_en_US.properties
b/plugins/transforms/textfile/src/main/resources/org/apache/hop/pipeline/transforms/csvinput/messages/messages_en_US.properties
index bece27ba92..d61be101e3 100644
---
a/plugins/transforms/textfile/src/main/resources/org/apache/hop/pipeline/transforms/csvinput/messages/messages_en_US.properties
+++
b/plugins/transforms/textfile/src/main/resources/org/apache/hop/pipeline/transforms/csvinput/messages/messages_en_US.properties
@@ -85,6 +85,11 @@
CsvInputMeta.CheckResult.NoInputReceivedFromOtherTransforms=No input received fr
CsvInputMeta.CheckResult.NotReceivingFields=Not receiving any fields from
previous transforms\!
CsvInputMeta.CheckResult.TransformRecevingData=Transform is connected to
previous one, receiving {0} fields
CsvInputMeta.CheckResult.TransformRecevingData2=Transform is receiving info
from other transforms.
+CsvInputMeta.CheckResult.FilenameMissing=No filename is specified to read from.
+CsvInputMeta.CheckResult.FilenameOk=Filename is specified.
+CsvInputMeta.CheckResult.FilenameFieldMissing=Filename field is not specified.
+CsvInputMeta.CheckResult.FilenameFieldNotFound=The filename field ''{0}''
could not be found in the previous transform.
+CsvInputMeta.CheckResult.FilenameFieldOk=The filename field ''{0}'' is present
in the previous transform.
CsvInputMeta.Injection.ADD_RESULT=Add files to result?
CsvInputMeta.Injection.BUFFER_SIZE=Read buffer size
CsvInputMeta.Injection.DELIMITER=Delimiter
diff --git
a/plugins/transforms/textfile/src/main/resources/org/apache/hop/pipeline/transforms/csvinput/messages/messages_zh_CN.properties
b/plugins/transforms/textfile/src/main/resources/org/apache/hop/pipeline/transforms/csvinput/messages/messages_zh_CN.properties
index 2a2880669c..48d5b96323 100644
---
a/plugins/transforms/textfile/src/main/resources/org/apache/hop/pipeline/transforms/csvinput/messages/messages_zh_CN.properties
+++
b/plugins/transforms/textfile/src/main/resources/org/apache/hop/pipeline/transforms/csvinput/messages/messages_zh_CN.properties
@@ -75,6 +75,11 @@
CsvInputMeta.CheckResult.NoInputReceivedFromOtherTransforms=\u6CA1\u6709\u4ECE\u
CsvInputMeta.CheckResult.NotReceivingFields=\u6CA1\u6709\u4ECE\u524D\u7F6E\u901A\u9053\u63A5\u6536\u5230\u4EFB\u4F55\u5B57\u6BB5
CsvInputMeta.CheckResult.TransformRecevingData=\u5DF2\u8FDE\u63A5\u5230\u524D\u7F6E\u901A\u9053,
\u63A5\u6536\u4E86 {0} \u4E2A\u5B57\u6BB5.
CsvInputMeta.CheckResult.TransformRecevingData2=\u6B63\u5728\u4ECE\u524D\u7F6E\u901A\u9053\u4E2D\u63A5\u6536\u4FE1\u606F
+CsvInputMeta.CheckResult.FilenameMissing=\u672A\u6307\u5B9A\u8981\u8BFB\u53D6\u7684\u6587\u4EF6\u540D\u3002
+CsvInputMeta.CheckResult.FilenameOk=\u5DF2\u6307\u5B9A\u6587\u4EF6\u540D\u3002
+CsvInputMeta.CheckResult.FilenameFieldMissing=\u672A\u6307\u5B9A\u6587\u4EF6\u540D\u5B57\u6BB5\u3002
+CsvInputMeta.CheckResult.FilenameFieldNotFound=\u524D\u7F6E\u901A\u9053\u4E2D\u627E\u4E0D\u5230\u6587\u4EF6\u540D\u5B57\u6BB5
''{0}''\u3002
+CsvInputMeta.CheckResult.FilenameFieldOk=\u524D\u7F6E\u901A\u9053\u4E2D\u5B58\u5728\u6587\u4EF6\u540D\u5B57\u6BB5
''{0}''\u3002
CsvInputMeta.Injection.ADD_RESULT=\u6DFB\u52A0\u6587\u4EF6\u5230\u7ED3\u679C?
CsvInputMeta.Injection.BUFFER_SIZE=\u8BFB\u53D6\u7F13\u51B2\u533A\u5927\u5C0F
CsvInputMeta.Injection.DELIMITER=\u5206\u9694\u7B26
diff --git
a/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputMetaTest.java
b/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputMetaTest.java
index 5b734ab6cf..5fc9744c45 100644
---
a/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputMetaTest.java
+++
b/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputMetaTest.java
@@ -26,11 +26,14 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import org.apache.hop.core.Const;
import org.apache.hop.core.HopClientEnvironment;
+import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.plugins.PluginRegistry;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowMeta;
@@ -45,6 +48,7 @@ import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.metadata.inject.HopMetadataInjector;
import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
import org.apache.hop.metadata.serializer.xml.XmlMetadataUtil;
+import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -165,4 +169,130 @@ class CsvInputMetaTest {
assertDoesNotThrow(() -> rowMeta.getString(row, 0));
assertEquals("abc", rowMeta.getString(row, 0));
}
+
+ @Test
+ void checkFilenameFromFieldDoesNotFlagIncomingHopsAsError() throws Exception
{
+ HopClientEnvironment.init();
+
+ CsvInputMeta meta = new CsvInputMeta();
+ meta.setFilenameField("csv_path");
+
+ RowMeta prev = new RowMeta();
+ prev.addValueMeta(new ValueMetaString("csv_path"));
+ List<ICheckResult> remarks = new ArrayList<>();
+
+ meta.check(
+ remarks,
+ new PipelineMeta(),
+ new TransformMeta(),
+ prev,
+ new String[] {"Generate filename"},
+ new String[0],
+ new RowMeta(),
+ new Variables(),
+ null);
+
+ assertEquals(0, errorCount(remarks));
+ }
+
+ @Test
+ void checkFilenameFromFieldReportsMissingField() throws Exception {
+ HopClientEnvironment.init();
+
+ CsvInputMeta meta = new CsvInputMeta();
+ meta.setFilenameField("csv_path");
+
+ RowMeta prev = new RowMeta();
+ prev.addValueMeta(new ValueMetaString("other"));
+ List<ICheckResult> remarks = new ArrayList<>();
+
+ meta.check(
+ remarks,
+ new PipelineMeta(),
+ new TransformMeta(),
+ prev,
+ new String[] {"Generate filename"},
+ new String[0],
+ new RowMeta(),
+ new Variables(),
+ null);
+
+ assertTrue(errorCount(remarks) >= 1);
+ assertTrue(
+ remarks.stream()
+ .anyMatch(
+ r ->
+ r.getType() == ICheckResult.TYPE_RESULT_ERROR
+ && r.getText().contains("csv_path")));
+ }
+
+ @Test
+ void checkFilenameFromFieldReportsMissingFilenameField() throws Exception {
+ HopClientEnvironment.init();
+
+ CsvInputMeta meta = new CsvInputMeta();
+
+ RowMeta prev = new RowMeta();
+ prev.addValueMeta(new ValueMetaString("csv_path"));
+ List<ICheckResult> remarks = new ArrayList<>();
+
+ meta.check(
+ remarks,
+ new PipelineMeta(),
+ new TransformMeta(),
+ prev,
+ new String[] {"Generate filename"},
+ new String[0],
+ new RowMeta(),
+ new Variables(),
+ null);
+
+ assertTrue(errorCount(remarks) >= 1);
+ }
+
+ @Test
+ void checkStaticFilenameWithNoInputIsOk() throws Exception {
+ HopClientEnvironment.init();
+
+ CsvInputMeta meta = new CsvInputMeta();
+ meta.setFilename("file.csv");
+
+ List<ICheckResult> remarks = new ArrayList<>();
+ meta.check(
+ remarks,
+ new PipelineMeta(),
+ new TransformMeta(),
+ new RowMeta(),
+ new String[0],
+ new String[0],
+ new RowMeta(),
+ new Variables(),
+ null);
+
+ assertEquals(0, errorCount(remarks));
+ }
+
+ @Test
+ void checkStaticFilenameReportsMissingFile() throws Exception {
+ HopClientEnvironment.init();
+
+ CsvInputMeta meta = new CsvInputMeta();
+ List<ICheckResult> remarks = new ArrayList<>();
+ meta.check(
+ remarks,
+ new PipelineMeta(),
+ new TransformMeta(),
+ new RowMeta(),
+ new String[0],
+ new String[0],
+ new RowMeta(),
+ new Variables(),
+ null);
+
+ assertTrue(errorCount(remarks) >= 1);
+ }
+
+ private static long errorCount(List<ICheckResult> remarks) {
+ return remarks.stream().filter(r -> r.getType() ==
ICheckResult.TYPE_RESULT_ERROR).count();
+ }
}
