This is an automated email from the ASF dual-hosted git repository. hansva pushed a commit to branch 2.18.1-patch in repository https://gitbox.apache.org/repos/asf/hop.git
commit fb16c8c7510641d0c208cd0480ec6d5698cf4804 Author: Hans Van Akelyen <[email protected]> AuthorDate: Sun Jun 7 20:39:15 2026 +0200 terafast and some other fixes, fixes #7239 (#7240) --- .../evalfilesmetrics/ActionEvalFilesMetrics.java | 4 +++ .../workflow/actions/sftpput/ActionSftpPut.java | 22 ++++++++++++++++ .../actions/sftpput/ActionSftpPutTest.java | 30 ++++++++++++++++++++++ .../loadfileinput/LoadFileInputDialog.java | 8 ++++-- .../mysqlbulkloader/MySqlBulkLoaderMeta.java | 10 ++++++++ .../mysqlbulkloader/MySqlBulkLoaderMetaTest.java | 9 +++++++ .../transforms/systemdata/SystemDataMeta.java | 10 ++++++++ .../transforms/systemdata/SystemDataMetaTest.java | 29 +++++++++++++++++++++ .../pipeline/transforms/terafast/TeraFastMeta.java | 5 ++-- .../transforms/terafast/TeraFastMetaTest.java | 16 ++++++++++++ 10 files changed, 139 insertions(+), 4 deletions(-) diff --git a/plugins/actions/evalfilesmetrics/src/main/java/org/apache/hop/workflow/actions/evalfilesmetrics/ActionEvalFilesMetrics.java b/plugins/actions/evalfilesmetrics/src/main/java/org/apache/hop/workflow/actions/evalfilesmetrics/ActionEvalFilesMetrics.java index b6473fc90d..4e645b85bf 100644 --- a/plugins/actions/evalfilesmetrics/src/main/java/org/apache/hop/workflow/actions/evalfilesmetrics/ActionEvalFilesMetrics.java +++ b/plugins/actions/evalfilesmetrics/src/main/java/org/apache/hop/workflow/actions/evalfilesmetrics/ActionEvalFilesMetrics.java @@ -654,6 +654,10 @@ public class ActionEvalFilesMetrics extends ActionBase implements Cloneable, IAc private void getFileSize(FileObject file) { try { + // Count every processed file (both SIZE and COUNT evaluation types); this running total is + // surfaced in the logs. The @HopMetadataProperty rewrite dropped this increment, leaving the + // reported file count stuck at 0. + realFilesCount = realFilesCount.add(ONE); if (isDetailed()) { logDetailed( BaseMessages.getString( diff --git a/plugins/actions/ftp/src/main/java/org/apache/hop/workflow/actions/sftpput/ActionSftpPut.java b/plugins/actions/ftp/src/main/java/org/apache/hop/workflow/actions/sftpput/ActionSftpPut.java index 12dd50b76d..90b280588f 100644 --- a/plugins/actions/ftp/src/main/java/org/apache/hop/workflow/actions/sftpput/ActionSftpPut.java +++ b/plugins/actions/ftp/src/main/java/org/apache/hop/workflow/actions/sftpput/ActionSftpPut.java @@ -35,9 +35,11 @@ import org.apache.hop.core.RowMetaAndData; import org.apache.hop.core.annotations.Action; import org.apache.hop.core.encryption.Encr; import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopXmlException; import org.apache.hop.core.util.Utils; import org.apache.hop.core.variables.IVariables; import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.core.xml.XmlHandler; import org.apache.hop.i18n.BaseMessages; import org.apache.hop.metadata.api.HopMetadataProperty; import org.apache.hop.metadata.api.IEnumHasCodeAndDescription; @@ -51,6 +53,7 @@ import org.apache.hop.workflow.action.IAction; import org.apache.hop.workflow.action.validator.ActionValidatorUtils; import org.apache.hop.workflow.action.validator.AndValidator; import org.apache.hop.workflow.actions.sftp.SftpClient; +import org.w3c.dom.Node; /** This defines an SFTP put action. */ @Action( @@ -183,6 +186,25 @@ public class ActionSftpPut extends ActionBase implements Cloneable, IAction { this.successWhenNoFile = a.successWhenNoFile; } + @Override + public void loadXml(Node entryNode, IHopMetadataProvider metadataProvider, IVariables variables) + throws HopXmlException { + super.loadXml(entryNode, metadataProvider, variables); + + // A missing/empty <aftersftpput> deserializes the enum to null; default it to NOTHING so the + // dialog and the execute() switch never hit a NullPointerException on legacy pipelines. + if (afterSftpAction == null) { + afterSftpAction = AfterFtpAction.NOTHING; + } + + // Backward compatibility: pre-2.18 files stored "delete after put" as <remove>Y</remove> + // instead of <aftersftpput>delete</aftersftpput>. Promote it so those workflows keep deleting. + if (afterSftpAction == AfterFtpAction.NOTHING + && "Y".equalsIgnoreCase(XmlHandler.getTagValue(entryNode, "remove"))) { + afterSftpAction = AfterFtpAction.DELETE; + } + } + @Override public Object clone() { return new ActionSftpPut(this); diff --git a/plugins/actions/ftp/src/test/java/org/apache/hop/workflow/actions/sftpput/ActionSftpPutTest.java b/plugins/actions/ftp/src/test/java/org/apache/hop/workflow/actions/sftpput/ActionSftpPutTest.java index 0e08f3e69a..7e522cc06b 100644 --- a/plugins/actions/ftp/src/test/java/org/apache/hop/workflow/actions/sftpput/ActionSftpPutTest.java +++ b/plugins/actions/ftp/src/test/java/org/apache/hop/workflow/actions/sftpput/ActionSftpPutTest.java @@ -26,9 +26,13 @@ import org.apache.hop.core.encryption.HopTwoWayPasswordEncoder; import org.apache.hop.core.encryption.TwoWayPasswordEncoderPlugin; import org.apache.hop.core.encryption.TwoWayPasswordEncoderPluginType; import org.apache.hop.core.plugins.PluginRegistry; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.core.xml.XmlHandler; +import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider; import org.apache.hop.workflow.action.ActionSerializationTestUtil; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.w3c.dom.Node; class ActionSftpPutTest { @BeforeEach @@ -72,4 +76,30 @@ class ActionSftpPutTest { assertTrue(action.isPreserveTargetFileTimestamp()); assertTrue(action.isSuccessWhenNoFile()); } + + @Test + void testLegacyRemoveTagPromotedToDelete() throws Exception { + // Pre-2.18 files stored "delete after put" as <remove>Y</remove>. After the + // @HopMetadataProperty + // migration that tag is no longer mapped, so it must be promoted to AfterFtpAction.DELETE on + // load, otherwise those workflows silently stop deleting the source files. + ActionSftpPut action = loadFromXml("<action><remove>Y</remove></action>"); + assertEquals(ActionSftpPut.AfterFtpAction.DELETE, action.getAfterSftpAction()); + } + + @Test + void testMissingAfterActionDefaultsToNothingNotNull() throws Exception { + // A legacy file without <aftersftpput> must not leave the enum null (it would NPE the dialog + // and + // the execute() switch). + ActionSftpPut action = loadFromXml("<action></action>"); + assertEquals(ActionSftpPut.AfterFtpAction.NOTHING, action.getAfterSftpAction()); + } + + private static ActionSftpPut loadFromXml(String xml) throws Exception { + Node node = XmlHandler.getSubNode(XmlHandler.loadXmlString(xml), "action"); + ActionSftpPut action = new ActionSftpPut(); + action.loadXml(node, new MemoryMetadataProvider(), new Variables()); + return action; + } } diff --git a/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInputDialog.java b/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInputDialog.java index 8d8c484e60..6f2767b833 100644 --- a/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInputDialog.java +++ b/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInputDialog.java @@ -1158,8 +1158,12 @@ public class LoadFileInputDialog extends BaseTransformDialog { inputFile.getFileName(), inputFile.getFileMask(), inputFile.getExcludeFileMask(), - inputFile.isFileRequired() ? CONST_COMBO_YES : CONST_COMBO_NO, - inputFile.isIncludeSubFolders() ? CONST_COMBO_YES : CONST_COMBO_NO); + inputFile.isFileRequired() + ? BaseMessages.getString(PKG, CONST_COMBO_YES) + : BaseMessages.getString(PKG, CONST_COMBO_NO), + inputFile.isIncludeSubFolders() + ? BaseMessages.getString(PKG, CONST_COMBO_YES) + : BaseMessages.getString(PKG, CONST_COMBO_NO)); } wFilenameList.optimizeTableView(); diff --git a/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMeta.java b/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMeta.java index dec0d3f8bb..bd02036dcf 100644 --- a/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMeta.java +++ b/plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMeta.java @@ -497,6 +497,16 @@ public class MySqlBulkLoaderMeta extends BaseTransformMeta<MySqlBulkLoader, MySq IEnumHasCodeAndDescription.lookupDescription( FieldFormatType.class, description, FieldFormatType.OK); } + + /** + * A missing or empty {@code <field_format_ok>} (legacy, injection-generated or hand-edited + * fields) must behave as {@link FieldFormatType#OK}, matching the pre-@HopMetadataProperty + * String default. Returning {@code null} caused a dialog NPE on open and changed the bytes + * written to the bulk-load file at runtime. + */ + public FieldFormatType getFieldFormatType() { + return fieldFormatType == null ? FieldFormatType.OK : fieldFormatType; + } } /** Added for backwards compatibility with older XML "mapping" blocks. */ diff --git a/plugins/transforms/mysqlbulkloader/src/test/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMetaTest.java b/plugins/transforms/mysqlbulkloader/src/test/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMetaTest.java index 651327c86b..583be1ff2d 100644 --- a/plugins/transforms/mysqlbulkloader/src/test/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMetaTest.java +++ b/plugins/transforms/mysqlbulkloader/src/test/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMetaTest.java @@ -26,6 +26,15 @@ import org.junit.jupiter.api.Test; class MySqlBulkLoaderMetaTest { + @Test + void testMissingFieldFormatDefaultsToOk() { + // Regression: a field with no/empty <field_format_ok> (legacy, injection-generated or + // hand-edited fields) must behave as OK, not null. Null caused a dialog NPE on open and changed + // the bytes written to the bulk-load file at runtime. + MySqlBulkLoaderMeta.Field field = new MySqlBulkLoaderMeta.Field(); + assertEquals(MySqlBulkLoaderMeta.FieldFormatType.OK, field.getFieldFormatType()); + } + @Test void testNewSerialization() throws Exception { MySqlBulkLoaderMeta meta = diff --git a/plugins/transforms/systemdata/src/main/java/org/apache/hop/pipeline/transforms/systemdata/SystemDataMeta.java b/plugins/transforms/systemdata/src/main/java/org/apache/hop/pipeline/transforms/systemdata/SystemDataMeta.java index b40491a6bb..1a9890eed6 100644 --- a/plugins/transforms/systemdata/src/main/java/org/apache/hop/pipeline/transforms/systemdata/SystemDataMeta.java +++ b/plugins/transforms/systemdata/src/main/java/org/apache/hop/pipeline/transforms/systemdata/SystemDataMeta.java @@ -244,5 +244,15 @@ public class SystemDataMeta extends BaseTransformMeta<SystemData, SystemDataData this.fieldName = f.fieldName; this.fieldType = f.fieldType; } + + /** + * A missing or unrecognized {@code <type>} (e.g. legacy or hand-edited pipelines) must behave + * as {@link SystemDataType#NONE}, matching the pre-@HopMetadataProperty {@code + * getTypeFromString()} fallback. Returning {@code null} caused NPEs in {@link + * SystemDataMeta#getFields} and {@link SystemDataMeta#check} and in the dialog. + */ + public SystemDataType getFieldType() { + return fieldType == null ? SystemDataType.NONE : fieldType; + } } } diff --git a/plugins/transforms/systemdata/src/test/java/org/apache/hop/pipeline/transforms/systemdata/SystemDataMetaTest.java b/plugins/transforms/systemdata/src/test/java/org/apache/hop/pipeline/transforms/systemdata/SystemDataMetaTest.java index 82d58801c2..740cb00e17 100644 --- a/plugins/transforms/systemdata/src/test/java/org/apache/hop/pipeline/transforms/systemdata/SystemDataMetaTest.java +++ b/plugins/transforms/systemdata/src/test/java/org/apache/hop/pipeline/transforms/systemdata/SystemDataMetaTest.java @@ -68,6 +68,35 @@ class SystemDataMetaTest { validate(metaCopy); } + @Test + void testFieldWithMissingTypeDefaultsToNoneAndDoesNotThrow() throws Exception { + // Regression: a field whose <type> is missing/empty (legacy or hand-edited pipelines) used to + // map to NONE. The enum migration made it deserialize to null, causing NPEs in getFields(), + // check() and the dialog. getFieldType() must never return null. + String xml = + XmlHandler.openTag(TransformMeta.XML_TAG) + + "<fields><field><name>legacy</name></field></fields>" + + XmlHandler.closeTag(TransformMeta.XML_TAG); + SystemDataMeta meta = new SystemDataMeta(); + XmlMetadataUtil.deSerializeFromXml( + XmlHandler.loadXmlString(xml, TransformMeta.XML_TAG), + SystemDataMeta.class, + meta, + new MemoryMetadataProvider()); + + assertEquals(1, meta.getFields().size()); + assertEquals(SystemDataType.NONE, meta.getFields().getFirst().getFieldType()); + + // None of the consumers below must throw a NullPointerException. + RowMeta rowMeta = new RowMeta(); + meta.getFields(rowMeta, "t", null, null, new Variables(), null); + assertEquals(IValueMeta.TYPE_NONE, rowMeta.getValueMeta(0).getType()); + + List<ICheckResult> remarks = new ArrayList<>(); + meta.check(remarks, null, null, null, null, null, null, new Variables(), null); + assertFalse(remarks.isEmpty()); + } + @Test void testGetFieldsDefaultType() throws Exception { SystemDataMeta meta = new SystemDataMeta(); diff --git a/plugins/transforms/terafast/src/main/java/org/apache/hop/pipeline/transforms/terafast/TeraFastMeta.java b/plugins/transforms/terafast/src/main/java/org/apache/hop/pipeline/transforms/terafast/TeraFastMeta.java index b51e863ec2..e9f4184eec 100644 --- a/plugins/transforms/terafast/src/main/java/org/apache/hop/pipeline/transforms/terafast/TeraFastMeta.java +++ b/plugins/transforms/terafast/src/main/java/org/apache/hop/pipeline/transforms/terafast/TeraFastMeta.java @@ -17,6 +17,7 @@ package org.apache.hop.pipeline.transforms.terafast; +import java.util.ArrayList; import java.util.List; import lombok.Getter; import lombok.Setter; @@ -108,10 +109,10 @@ public class TeraFastMeta extends BaseTransformMeta<ITransform, ITransformData> private String targetTable; @HopMetadataProperty(key = "table_field_list") - private List<String> tableFieldList; + private List<String> tableFieldList = new ArrayList<>(); @HopMetadataProperty(key = "stream_field_list") - private List<String> streamFieldList; + private List<String> streamFieldList = new ArrayList<>(); @HopMetadataProperty( key = "connectionName", diff --git a/plugins/transforms/terafast/src/test/java/org/apache/hop/pipeline/transforms/terafast/TeraFastMetaTest.java b/plugins/transforms/terafast/src/test/java/org/apache/hop/pipeline/transforms/terafast/TeraFastMetaTest.java index 77c45fe6d2..44ba3b1ac9 100644 --- a/plugins/transforms/terafast/src/test/java/org/apache/hop/pipeline/transforms/terafast/TeraFastMetaTest.java +++ b/plugins/transforms/terafast/src/test/java/org/apache/hop/pipeline/transforms/terafast/TeraFastMetaTest.java @@ -59,6 +59,22 @@ class TeraFastMetaTest { assertTrue(meta.isUseControlFile()); } + @Test + void testFieldListsAreNeverNullAfterConstructionAndSetDefault() { + // Regression: the @HopMetadataProperty refactor dropped the constructor initialization of these + // lists, leaving them null after construction/setDefault(). That caused NPEs when opening the + // dialog, running check(), or executing processRow() on a freshly added TeraFast transform. + TeraFastMeta meta = new TeraFastMeta(); + assertNotNull(meta.getTableFieldList()); + assertNotNull(meta.getStreamFieldList()); + + meta.setDefault(); + assertNotNull(meta.getTableFieldList()); + assertNotNull(meta.getStreamFieldList()); + assertTrue(meta.getTableFieldList().isEmpty()); + assertTrue(meta.getStreamFieldList().isEmpty()); + } + @Test void testClone() { TeraFastMeta meta = new TeraFastMeta();
