This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new aa1a25ffb2 terafast and some other fixes, fixes #7239 (#7240)
aa1a25ffb2 is described below
commit aa1a25ffb2c42c90551417376bef3abdf613d91b
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Sun Jun 7 20:39:15 2026 +0200
terafast and some other fixes, fixes #7239 (#7240)
---
.../evalfilesmetrics/ActionEvalFilesMetrics.java | 4 +++
.../workflow/actions/sftpput/ActionSftpPut.java | 22 ++++++++++++++++
.../actions/sftpput/ActionSftpPutTest.java | 30 ++++++++++++++++++++++
.../loadfileinput/LoadFileInputDialog.java | 8 ++++--
.../mysqlbulkloader/MySqlBulkLoaderMeta.java | 10 ++++++++
.../mysqlbulkloader/MySqlBulkLoaderMetaTest.java | 9 +++++++
.../transforms/systemdata/SystemDataMeta.java | 10 ++++++++
.../transforms/systemdata/SystemDataMetaTest.java | 29 +++++++++++++++++++++
.../pipeline/transforms/terafast/TeraFastMeta.java | 5 ++--
.../transforms/terafast/TeraFastMetaTest.java | 16 ++++++++++++
10 files changed, 139 insertions(+), 4 deletions(-)
diff --git
a/plugins/actions/evalfilesmetrics/src/main/java/org/apache/hop/workflow/actions/evalfilesmetrics/ActionEvalFilesMetrics.java
b/plugins/actions/evalfilesmetrics/src/main/java/org/apache/hop/workflow/actions/evalfilesmetrics/ActionEvalFilesMetrics.java
index b6473fc90d..4e645b85bf 100644
---
a/plugins/actions/evalfilesmetrics/src/main/java/org/apache/hop/workflow/actions/evalfilesmetrics/ActionEvalFilesMetrics.java
+++
b/plugins/actions/evalfilesmetrics/src/main/java/org/apache/hop/workflow/actions/evalfilesmetrics/ActionEvalFilesMetrics.java
@@ -654,6 +654,10 @@ public class ActionEvalFilesMetrics extends ActionBase
implements Cloneable, IAc
private void getFileSize(FileObject file) {
try {
+ // Count every processed file (both SIZE and COUNT evaluation types);
this running total is
+ // surfaced in the logs. The @HopMetadataProperty rewrite dropped this
increment, leaving the
+ // reported file count stuck at 0.
+ realFilesCount = realFilesCount.add(ONE);
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
diff --git
a/plugins/actions/ftp/src/main/java/org/apache/hop/workflow/actions/sftpput/ActionSftpPut.java
b/plugins/actions/ftp/src/main/java/org/apache/hop/workflow/actions/sftpput/ActionSftpPut.java
index 12dd50b76d..90b280588f 100644
---
a/plugins/actions/ftp/src/main/java/org/apache/hop/workflow/actions/sftpput/ActionSftpPut.java
+++
b/plugins/actions/ftp/src/main/java/org/apache/hop/workflow/actions/sftpput/ActionSftpPut.java
@@ -35,9 +35,11 @@ import org.apache.hop.core.RowMetaAndData;
import org.apache.hop.core.annotations.Action;
import org.apache.hop.core.encryption.Encr;
import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopXmlException;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.metadata.api.IEnumHasCodeAndDescription;
@@ -51,6 +53,7 @@ import org.apache.hop.workflow.action.IAction;
import org.apache.hop.workflow.action.validator.ActionValidatorUtils;
import org.apache.hop.workflow.action.validator.AndValidator;
import org.apache.hop.workflow.actions.sftp.SftpClient;
+import org.w3c.dom.Node;
/** This defines an SFTP put action. */
@Action(
@@ -183,6 +186,25 @@ public class ActionSftpPut extends ActionBase implements
Cloneable, IAction {
this.successWhenNoFile = a.successWhenNoFile;
}
+ @Override
+ public void loadXml(Node entryNode, IHopMetadataProvider metadataProvider,
IVariables variables)
+ throws HopXmlException {
+ super.loadXml(entryNode, metadataProvider, variables);
+
+ // A missing/empty <aftersftpput> deserializes the enum to null; default
it to NOTHING so the
+ // dialog and the execute() switch never hit a NullPointerException on
legacy pipelines.
+ if (afterSftpAction == null) {
+ afterSftpAction = AfterFtpAction.NOTHING;
+ }
+
+ // Backward compatibility: pre-2.18 files stored "delete after put" as
<remove>Y</remove>
+ // instead of <aftersftpput>delete</aftersftpput>. Promote it so those
workflows keep deleting.
+ if (afterSftpAction == AfterFtpAction.NOTHING
+ && "Y".equalsIgnoreCase(XmlHandler.getTagValue(entryNode, "remove"))) {
+ afterSftpAction = AfterFtpAction.DELETE;
+ }
+ }
+
@Override
public Object clone() {
return new ActionSftpPut(this);
diff --git
a/plugins/actions/ftp/src/test/java/org/apache/hop/workflow/actions/sftpput/ActionSftpPutTest.java
b/plugins/actions/ftp/src/test/java/org/apache/hop/workflow/actions/sftpput/ActionSftpPutTest.java
index 0e08f3e69a..7e522cc06b 100644
---
a/plugins/actions/ftp/src/test/java/org/apache/hop/workflow/actions/sftpput/ActionSftpPutTest.java
+++
b/plugins/actions/ftp/src/test/java/org/apache/hop/workflow/actions/sftpput/ActionSftpPutTest.java
@@ -26,9 +26,13 @@ import
org.apache.hop.core.encryption.HopTwoWayPasswordEncoder;
import org.apache.hop.core.encryption.TwoWayPasswordEncoderPlugin;
import org.apache.hop.core.encryption.TwoWayPasswordEncoderPluginType;
import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.core.xml.XmlHandler;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
import org.apache.hop.workflow.action.ActionSerializationTestUtil;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.w3c.dom.Node;
class ActionSftpPutTest {
@BeforeEach
@@ -72,4 +76,30 @@ class ActionSftpPutTest {
assertTrue(action.isPreserveTargetFileTimestamp());
assertTrue(action.isSuccessWhenNoFile());
}
+
+ @Test
+ void testLegacyRemoveTagPromotedToDelete() throws Exception {
+ // Pre-2.18 files stored "delete after put" as <remove>Y</remove>. After
the
+ // @HopMetadataProperty
+ // migration that tag is no longer mapped, so it must be promoted to
AfterFtpAction.DELETE on
+ // load, otherwise those workflows silently stop deleting the source files.
+ ActionSftpPut action = loadFromXml("<action><remove>Y</remove></action>");
+ assertEquals(ActionSftpPut.AfterFtpAction.DELETE,
action.getAfterSftpAction());
+ }
+
+ @Test
+ void testMissingAfterActionDefaultsToNothingNotNull() throws Exception {
+ // A legacy file without <aftersftpput> must not leave the enum null (it
would NPE the dialog
+ // and
+ // the execute() switch).
+ ActionSftpPut action = loadFromXml("<action></action>");
+ assertEquals(ActionSftpPut.AfterFtpAction.NOTHING,
action.getAfterSftpAction());
+ }
+
+ private static ActionSftpPut loadFromXml(String xml) throws Exception {
+ Node node = XmlHandler.getSubNode(XmlHandler.loadXmlString(xml), "action");
+ ActionSftpPut action = new ActionSftpPut();
+ action.loadXml(node, new MemoryMetadataProvider(), new Variables());
+ return action;
+ }
}
diff --git
a/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInputDialog.java
b/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInputDialog.java
index 8d8c484e60..6f2767b833 100644
---
a/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInputDialog.java
+++
b/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInputDialog.java
@@ -1158,8 +1158,12 @@ public class LoadFileInputDialog extends
BaseTransformDialog {
inputFile.getFileName(),
inputFile.getFileMask(),
inputFile.getExcludeFileMask(),
- inputFile.isFileRequired() ? CONST_COMBO_YES : CONST_COMBO_NO,
- inputFile.isIncludeSubFolders() ? CONST_COMBO_YES : CONST_COMBO_NO);
+ inputFile.isFileRequired()
+ ? BaseMessages.getString(PKG, CONST_COMBO_YES)
+ : BaseMessages.getString(PKG, CONST_COMBO_NO),
+ inputFile.isIncludeSubFolders()
+ ? BaseMessages.getString(PKG, CONST_COMBO_YES)
+ : BaseMessages.getString(PKG, CONST_COMBO_NO));
}
wFilenameList.optimizeTableView();
diff --git
a/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMeta.java
b/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMeta.java
index dec0d3f8bb..bd02036dcf 100644
---
a/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMeta.java
+++
b/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMeta.java
@@ -497,6 +497,16 @@ public class MySqlBulkLoaderMeta extends
BaseTransformMeta<MySqlBulkLoader, MySq
IEnumHasCodeAndDescription.lookupDescription(
FieldFormatType.class, description, FieldFormatType.OK);
}
+
+ /**
+ * A missing or empty {@code <field_format_ok>} (legacy,
injection-generated or hand-edited
+ * fields) must behave as {@link FieldFormatType#OK}, matching the
pre-@HopMetadataProperty
+ * String default. Returning {@code null} caused a dialog NPE on open and
changed the bytes
+ * written to the bulk-load file at runtime.
+ */
+ public FieldFormatType getFieldFormatType() {
+ return fieldFormatType == null ? FieldFormatType.OK : fieldFormatType;
+ }
}
/** Added for backwards compatibility with older XML "mapping" blocks. */
diff --git
a/plugins/transforms/mysqlbulkloader/src/test/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMetaTest.java
b/plugins/transforms/mysqlbulkloader/src/test/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMetaTest.java
index 651327c86b..583be1ff2d 100644
---
a/plugins/transforms/mysqlbulkloader/src/test/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMetaTest.java
+++
b/plugins/transforms/mysqlbulkloader/src/test/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMetaTest.java
@@ -26,6 +26,15 @@ import org.junit.jupiter.api.Test;
class MySqlBulkLoaderMetaTest {
+ @Test
+ void testMissingFieldFormatDefaultsToOk() {
+ // Regression: a field with no/empty <field_format_ok> (legacy,
injection-generated or
+ // hand-edited fields) must behave as OK, not null. Null caused a dialog
NPE on open and changed
+ // the bytes written to the bulk-load file at runtime.
+ MySqlBulkLoaderMeta.Field field = new MySqlBulkLoaderMeta.Field();
+ assertEquals(MySqlBulkLoaderMeta.FieldFormatType.OK,
field.getFieldFormatType());
+ }
+
@Test
void testNewSerialization() throws Exception {
MySqlBulkLoaderMeta meta =
diff --git
a/plugins/transforms/systemdata/src/main/java/org/apache/hop/pipeline/transforms/systemdata/SystemDataMeta.java
b/plugins/transforms/systemdata/src/main/java/org/apache/hop/pipeline/transforms/systemdata/SystemDataMeta.java
index b40491a6bb..1a9890eed6 100644
---
a/plugins/transforms/systemdata/src/main/java/org/apache/hop/pipeline/transforms/systemdata/SystemDataMeta.java
+++
b/plugins/transforms/systemdata/src/main/java/org/apache/hop/pipeline/transforms/systemdata/SystemDataMeta.java
@@ -244,5 +244,15 @@ public class SystemDataMeta extends
BaseTransformMeta<SystemData, SystemDataData
this.fieldName = f.fieldName;
this.fieldType = f.fieldType;
}
+
+ /**
+ * A missing or unrecognized {@code <type>} (e.g. legacy or hand-edited
pipelines) must behave
+ * as {@link SystemDataType#NONE}, matching the pre-@HopMetadataProperty
{@code
+ * getTypeFromString()} fallback. Returning {@code null} caused NPEs in
{@link
+ * SystemDataMeta#getFields} and {@link SystemDataMeta#check} and in the
dialog.
+ */
+ public SystemDataType getFieldType() {
+ return fieldType == null ? SystemDataType.NONE : fieldType;
+ }
}
}
diff --git
a/plugins/transforms/systemdata/src/test/java/org/apache/hop/pipeline/transforms/systemdata/SystemDataMetaTest.java
b/plugins/transforms/systemdata/src/test/java/org/apache/hop/pipeline/transforms/systemdata/SystemDataMetaTest.java
index 82d58801c2..740cb00e17 100644
---
a/plugins/transforms/systemdata/src/test/java/org/apache/hop/pipeline/transforms/systemdata/SystemDataMetaTest.java
+++
b/plugins/transforms/systemdata/src/test/java/org/apache/hop/pipeline/transforms/systemdata/SystemDataMetaTest.java
@@ -68,6 +68,35 @@ class SystemDataMetaTest {
validate(metaCopy);
}
+ @Test
+ void testFieldWithMissingTypeDefaultsToNoneAndDoesNotThrow() throws
Exception {
+ // Regression: a field whose <type> is missing/empty (legacy or
hand-edited pipelines) used to
+ // map to NONE. The enum migration made it deserialize to null, causing
NPEs in getFields(),
+ // check() and the dialog. getFieldType() must never return null.
+ String xml =
+ XmlHandler.openTag(TransformMeta.XML_TAG)
+ + "<fields><field><name>legacy</name></field></fields>"
+ + XmlHandler.closeTag(TransformMeta.XML_TAG);
+ SystemDataMeta meta = new SystemDataMeta();
+ XmlMetadataUtil.deSerializeFromXml(
+ XmlHandler.loadXmlString(xml, TransformMeta.XML_TAG),
+ SystemDataMeta.class,
+ meta,
+ new MemoryMetadataProvider());
+
+ assertEquals(1, meta.getFields().size());
+ assertEquals(SystemDataType.NONE,
meta.getFields().getFirst().getFieldType());
+
+ // None of the consumers below must throw a NullPointerException.
+ RowMeta rowMeta = new RowMeta();
+ meta.getFields(rowMeta, "t", null, null, new Variables(), null);
+ assertEquals(IValueMeta.TYPE_NONE, rowMeta.getValueMeta(0).getType());
+
+ List<ICheckResult> remarks = new ArrayList<>();
+ meta.check(remarks, null, null, null, null, null, null, new Variables(),
null);
+ assertFalse(remarks.isEmpty());
+ }
+
@Test
void testGetFieldsDefaultType() throws Exception {
SystemDataMeta meta = new SystemDataMeta();
diff --git
a/plugins/transforms/terafast/src/main/java/org/apache/hop/pipeline/transforms/terafast/TeraFastMeta.java
b/plugins/transforms/terafast/src/main/java/org/apache/hop/pipeline/transforms/terafast/TeraFastMeta.java
index b51e863ec2..e9f4184eec 100644
---
a/plugins/transforms/terafast/src/main/java/org/apache/hop/pipeline/transforms/terafast/TeraFastMeta.java
+++
b/plugins/transforms/terafast/src/main/java/org/apache/hop/pipeline/transforms/terafast/TeraFastMeta.java
@@ -17,6 +17,7 @@
package org.apache.hop.pipeline.transforms.terafast;
+import java.util.ArrayList;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
@@ -108,10 +109,10 @@ public class TeraFastMeta extends
BaseTransformMeta<ITransform, ITransformData>
private String targetTable;
@HopMetadataProperty(key = "table_field_list")
- private List<String> tableFieldList;
+ private List<String> tableFieldList = new ArrayList<>();
@HopMetadataProperty(key = "stream_field_list")
- private List<String> streamFieldList;
+ private List<String> streamFieldList = new ArrayList<>();
@HopMetadataProperty(
key = "connectionName",
diff --git
a/plugins/transforms/terafast/src/test/java/org/apache/hop/pipeline/transforms/terafast/TeraFastMetaTest.java
b/plugins/transforms/terafast/src/test/java/org/apache/hop/pipeline/transforms/terafast/TeraFastMetaTest.java
index 77c45fe6d2..44ba3b1ac9 100644
---
a/plugins/transforms/terafast/src/test/java/org/apache/hop/pipeline/transforms/terafast/TeraFastMetaTest.java
+++
b/plugins/transforms/terafast/src/test/java/org/apache/hop/pipeline/transforms/terafast/TeraFastMetaTest.java
@@ -59,6 +59,22 @@ class TeraFastMetaTest {
assertTrue(meta.isUseControlFile());
}
+ @Test
+ void testFieldListsAreNeverNullAfterConstructionAndSetDefault() {
+ // Regression: the @HopMetadataProperty refactor dropped the constructor
initialization of these
+ // lists, leaving them null after construction/setDefault(). That caused
NPEs when opening the
+ // dialog, running check(), or executing processRow() on a freshly added
TeraFast transform.
+ TeraFastMeta meta = new TeraFastMeta();
+ assertNotNull(meta.getTableFieldList());
+ assertNotNull(meta.getStreamFieldList());
+
+ meta.setDefault();
+ assertNotNull(meta.getTableFieldList());
+ assertNotNull(meta.getStreamFieldList());
+ assertTrue(meta.getTableFieldList().isEmpty());
+ assertTrue(meta.getStreamFieldList().isEmpty());
+ }
+
@Test
void testClone() {
TeraFastMeta meta = new TeraFastMeta();
