This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 2bf31583c7 fix #6071, fix #6274 (#6278)
2bf31583c7 is described below
commit 2bf31583c73d500c642e6c1007b098985b98c5ad
Author: Sergio Ramazzina <[email protected]>
AuthorDate: Wed Dec 31 09:03:29 2025 +0100
fix #6071, fix #6274 (#6278)
* Fix #6274 - MinIO plugin: MinIO part size parameter resolved from wrong
variable
* Fix #6071 - Get File Names don't list files in subfolder for minio
* Fixed compilation issues
* Updated integration tests to consider a new condition
---
...te-files.hpl => 0001-minio-validate-files1.hpl} | 0
...te-files.hpl => 0001-minio-validate-files2.hpl} | 13 ++++--
.../minio/main-0001-minio-file-handling.hwf | 46 ++++++++++++++++------
...T.json => 0001-minio-validate-files1 UNIT.json} | 4 +-
...T.json => 0001-minio-validate-files2 UNIT.json} | 4 +-
.../org/apache/hop/vfs/minio/MinioFileObject.java | 5 ++-
.../apache/hop/vfs/minio/MinioFileProvider.java | 5 ++-
7 files changed, 54 insertions(+), 23 deletions(-)
diff --git a/integration-tests/minio/0001-minio-validate-files.hpl
b/integration-tests/minio/0001-minio-validate-files1.hpl
similarity index 100%
copy from integration-tests/minio/0001-minio-validate-files.hpl
copy to integration-tests/minio/0001-minio-validate-files1.hpl
diff --git a/integration-tests/minio/0001-minio-validate-files.hpl
b/integration-tests/minio/0001-minio-validate-files2.hpl
similarity index 93%
rename from integration-tests/minio/0001-minio-validate-files.hpl
rename to integration-tests/minio/0001-minio-validate-files2.hpl
index d312808c4f..5ab93ef6a3 100644
--- a/integration-tests/minio/0001-minio-validate-files.hpl
+++ b/integration-tests/minio/0001-minio-validate-files2.hpl
@@ -19,7 +19,7 @@ limitations under the License.
-->
<pipeline>
<info>
- <name>0001-minio-validate-files</name>
+ <name>0001-minio-validate-files2</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@@ -115,13 +115,16 @@ limitations under the License.
</partitioning>
<doNotFailIfNoFile>N</doNotFailIfNoFile>
<dynamic_include_subfolders>N</dynamic_include_subfolders>
+ <exclude_wildcard_Field/>
<file>
+ <exclude_filemask/>
<file_required>N</file_required>
- <filemask>.*</filemask>
- <include_subfolders>N</include_subfolders>
- <name>minio:///demo/copy/</name>
+ <filemask>.*.txt</filemask>
+ <include_subfolders>Y</include_subfolders>
+ <name>minio:///demo</name>
</file>
<filefield>N</filefield>
+ <filename_Field/>
<filter>
<filterfiletype>all_files</filterfiletype>
</filter>
@@ -129,6 +132,8 @@ limitations under the License.
<limit>0</limit>
<raiseAnExceptionIfNoFile>N</raiseAnExceptionIfNoFile>
<rownum>N</rownum>
+ <rownum_field/>
+ <wildcard_Field/>
<attributes/>
<GUI>
<xloc>80</xloc>
diff --git a/integration-tests/minio/main-0001-minio-file-handling.hwf
b/integration-tests/minio/main-0001-minio-file-handling.hwf
index df617633d1..1326d5f083 100644
--- a/integration-tests/minio/main-0001-minio-file-handling.hwf
+++ b/integration-tests/minio/main-0001-minio-file-handling.hwf
@@ -246,7 +246,7 @@ limitations under the License.
<attributes/>
<parallel>N</parallel>
<xloc>576</xloc>
- <yloc>304</yloc>
+ <yloc>416</yloc>
<attributes_hac/>
</action>
<action>
@@ -330,17 +330,17 @@ limitations under the License.
<attributes_hac/>
</action>
<action>
- <name>minio-get-filenames UNIT</name>
+ <name>minio-get-filenames UNIT 1</name>
<description/>
<type>RunPipelineTests</type>
<attributes/>
<test_names>
<test_name>
- <name>0001-minio-validate-files UNIT</name>
+ <name>0001-minio-validate-files1 UNIT</name>
</test_name>
</test_names>
<parallel>N</parallel>
- <xloc>576</xloc>
+ <xloc>464</xloc>
<yloc>224</yloc>
<attributes_hac/>
</action>
@@ -379,7 +379,22 @@ limitations under the License.
<filename>minio:///demo/copy/</filename>
<parallel>N</parallel>
<xloc>160</xloc>
- <yloc>304</yloc>
+ <yloc>416</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>minio-get-filenames UNIT 2</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0001-minio-validate-files2 UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>624</xloc>
+ <yloc>224</yloc>
<attributes_hac/>
</action>
</actions>
@@ -483,24 +498,31 @@ limitations under the License.
<unconditional>N</unconditional>
</hop>
<hop>
- <from>minio-get-filenames UNIT</from>
- <to>Delete minio:///demo folders</to>
+ <from>Copy files</from>
+ <to>minio-get-filenames UNIT 1</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
- <from>Copy files</from>
- <to>minio-get-filenames UNIT</to>
+ <from>minio:///demo/copy/ exists</from>
+ <to>Success</to>
+ <enabled>Y</enabled>
+ <evaluation>N</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>minio-get-filenames UNIT 1</from>
+ <to>minio-get-filenames UNIT 2</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
- <from>minio:///demo/copy/ exists</from>
- <to>Success</to>
+ <from>minio-get-filenames UNIT 2</from>
+ <to>Delete minio:///demo folders</to>
<enabled>Y</enabled>
- <evaluation>N</evaluation>
+ <evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
</hops>
diff --git
a/integration-tests/minio/metadata/unit-test/0001-minio-validate-files
UNIT.json
b/integration-tests/minio/metadata/unit-test/0001-minio-validate-files1
UNIT.json
similarity index 94%
copy from integration-tests/minio/metadata/unit-test/0001-minio-validate-files
UNIT.json
copy to integration-tests/minio/metadata/unit-test/0001-minio-validate-files1
UNIT.json
index 4c4bf095e6..f5c8bd884b 100644
--- a/integration-tests/minio/metadata/unit-test/0001-minio-validate-files
UNIT.json
+++ b/integration-tests/minio/metadata/unit-test/0001-minio-validate-files1
UNIT.json
@@ -77,7 +77,7 @@
}
],
"input_data_sets": [],
- "name": "0001-minio-validate-files UNIT",
+ "name": "0001-minio-validate-files1 UNIT",
"trans_test_tweaks": [],
- "pipeline_filename": "./0001-minio-validate-files.hpl"
+ "pipeline_filename": "./0001-minio-validate-files1.hpl"
}
\ No newline at end of file
diff --git
a/integration-tests/minio/metadata/unit-test/0001-minio-validate-files
UNIT.json
b/integration-tests/minio/metadata/unit-test/0001-minio-validate-files2
UNIT.json
similarity index 94%
rename from
integration-tests/minio/metadata/unit-test/0001-minio-validate-files UNIT.json
rename to integration-tests/minio/metadata/unit-test/0001-minio-validate-files2
UNIT.json
index 4c4bf095e6..33e879107a 100644
--- a/integration-tests/minio/metadata/unit-test/0001-minio-validate-files
UNIT.json
+++ b/integration-tests/minio/metadata/unit-test/0001-minio-validate-files2
UNIT.json
@@ -77,7 +77,7 @@
}
],
"input_data_sets": [],
- "name": "0001-minio-validate-files UNIT",
+ "name": "0001-minio-validate-files2 UNIT",
"trans_test_tweaks": [],
- "pipeline_filename": "./0001-minio-validate-files.hpl"
+ "pipeline_filename": "./0001-minio-validate-files2.hpl"
}
\ No newline at end of file
diff --git
a/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/MinioFileObject.java
b/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/MinioFileObject.java
index 1e197b48bd..a5aeb4dadb 100644
---
a/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/MinioFileObject.java
+++
b/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/MinioFileObject.java
@@ -128,7 +128,10 @@ public class MinioFileObject extends
AbstractFileObject<MinioFileSystem> {
path += DELIMITER;
}
// The regular case
- ListObjectsArgs args =
ListObjectsArgs.builder().bucket(bucketName).prefix(path).build();
+ ListObjectsArgs.Builder listObjectsArgsBuilder =
ListObjectsArgs.builder().bucket(bucketName);
+ if (!path.equals(DELIMITER)) listObjectsArgsBuilder.prefix(path);
+
+ ListObjectsArgs args = listObjectsArgsBuilder.build();
Iterable<Result<Item>> results =
fileSystem.getClient().listObjects(args);
for (Result<Item> result : results) {
Item item = result.get();
diff --git
a/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/MinioFileProvider.java
b/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/MinioFileProvider.java
index ccf06cdae1..32d9917d7d 100644
---
a/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/MinioFileProvider.java
+++
b/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/MinioFileProvider.java
@@ -41,6 +41,7 @@ public class MinioFileProvider extends
AbstractOriginatingFileProvider {
new UserAuthenticationData.Type[] {
UserAuthenticationData.USERNAME, UserAuthenticationData.PASSWORD
};
+ public static final int DEFAULT_ENDPOINT_PORT = 9000;
public static FileSystemOptions getDefaultFileSystemOptions() {
return defaultOptions;
@@ -92,7 +93,7 @@ public class MinioFileProvider extends
AbstractOriginatingFileProvider {
// The end point port
String endPointPort = variables.resolve(minioMeta.getEndPointPort());
- fileSystem.setEndPointPort(Const.toInt(endPointPort, 9000));
+ fileSystem.setEndPointPort(Const.toInt(endPointPort,
DEFAULT_ENDPOINT_PORT));
// Is the end point secure? (https)
boolean endPointSecure = minioMeta.isEndPointSecure();
@@ -125,7 +126,7 @@ public class MinioFileProvider extends
AbstractOriginatingFileProvider {
fileSystem.setRegion(region);
// The part size
- String partSize = variables.resolve(minioMeta.getRegion());
+ String partSize = variables.resolve(minioMeta.getPartSize());
fileSystem.setPartSize(Const.toLong(partSize, DEFAULT_PART_SIZE));
return fileSystem;
