This is an automated email from the ASF dual-hosted git repository. zhongjiajie pushed a commit to branch 3.2.1-release in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit c2ff260c1a7587ce4fa919e07f957a42d0ed2c42 Author: Jay Chung <[email protected]> AuthorDate: Tue Feb 6 23:56:03 2024 +0800 fix: Resource relate path invalid when tenant change (#15581) * fix can not get correct resource related path when user run workflow with differnet tenant of resource created * also fix can not get correct related path when we use `resource.storage.type=LOCAL` (cherry picked from commit bd83631955062929a81aaf9a095166ada3a02909) --- .../plugin/storage/api/StorageOperate.java | 13 ++++++++++- .../plugin/storage/hdfs/HdfsStorageOperator.java | 2 +- ...ratorFactory.java => LocalStorageOperator.java} | 27 ++++++++++------------ .../storage/hdfs/LocalStorageOperatorFactory.java | 2 +- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java index ab096808b0..945e361a09 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.plugin.storage.api; +import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_FILE; + import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.utils.PropertyUtils; @@ -72,7 +74,16 @@ public interface StorageOperate { */ default String getResourceFileName(String tenantCode, String fullName) { String resDir = getResDir(tenantCode); - return fullName.replaceFirst(resDir, ""); + String filenameReplaceResDir = fullName.replaceFirst(resDir, ""); + if (!filenameReplaceResDir.equals(fullName)) { + return filenameReplaceResDir; + } + + // Replace resource dir not effective in case of run workflow with different tenant from resource file's. + // this is backup solution to get related path, by split with RESOURCE_TYPE_FILE + return filenameReplaceResDir.contains(RESOURCE_TYPE_FILE) + ? filenameReplaceResDir.split(String.format("%s/", RESOURCE_TYPE_FILE))[1] + : filenameReplaceResDir; } /** diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java index f5891a5b92..4479c93a3d 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java @@ -71,7 +71,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; @Slf4j public class HdfsStorageOperator implements Closeable, StorageOperate { - private static HdfsStorageProperties hdfsProperties = new HdfsStorageProperties(); + protected static HdfsStorageProperties hdfsProperties = new HdfsStorageProperties(); private static final String HADOOP_UTILS_KEY = "HADOOP_UTILS_KEY"; private volatile boolean yarnEnabled = false; diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperator.java similarity index 53% copy from dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java copy to dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperator.java index 1e244b2137..173760b632 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperator.java @@ -17,26 +17,23 @@ package org.apache.dolphinscheduler.plugin.storage.hdfs; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperateFactory; -import org.apache.dolphinscheduler.plugin.storage.api.StorageType; +import lombok.extern.slf4j.Slf4j; -import com.google.auto.service.AutoService; +@Slf4j +public class LocalStorageOperator extends HdfsStorageOperator { -@AutoService(StorageOperateFactory.class) -public class LocalStorageOperatorFactory implements StorageOperateFactory { - - private static final String LOCAL_DEFAULT_FS = "file:/"; + public LocalStorageOperator() { + super(new HdfsStorageProperties()); + } - @Override - public StorageOperate createStorageOperate() { - HdfsStorageProperties hdfsStorageProperties = new HdfsStorageProperties(); - hdfsStorageProperties.setDefaultFS(LOCAL_DEFAULT_FS); - return new HdfsStorageOperator(hdfsStorageProperties); + public LocalStorageOperator(HdfsStorageProperties hdfsStorageProperties) { + super(hdfsStorageProperties); } @Override - public StorageType getStorageOperate() { - return StorageType.LOCAL; + public String getResourceFileName(String tenantCode, String fullName) { + // prefix schema `file:/` should be remove in local file mode + String fullNameRemoveSchema = fullName.replaceFirst(hdfsProperties.getDefaultFS(), ""); + return super.getResourceFileName(tenantCode, fullNameRemoveSchema); } } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java index 1e244b2137..5f44eca87b 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java @@ -32,7 +32,7 @@ public class LocalStorageOperatorFactory implements StorageOperateFactory { public StorageOperate createStorageOperate() { HdfsStorageProperties hdfsStorageProperties = new HdfsStorageProperties(); hdfsStorageProperties.setDefaultFS(LOCAL_DEFAULT_FS); - return new HdfsStorageOperator(hdfsStorageProperties); + return new LocalStorageOperator(hdfsStorageProperties); } @Override
