This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new bd83631955 fix: Resource relate path invalid when tenant change
(#15581)
bd83631955 is described below
commit bd83631955062929a81aaf9a095166ada3a02909
Author: Jay Chung <[email protected]>
AuthorDate: Tue Feb 6 23:56:03 2024 +0800
fix: Resource relate path invalid when tenant change (#15581)
* fix can not get correct resource related path when
user run workflow with differnet tenant of resource
created
* also fix can not get correct related path when we
use `resource.storage.type=LOCAL`
---
.../plugin/storage/api/StorageOperate.java | 13 ++++++++++-
.../plugin/storage/hdfs/HdfsStorageOperator.java | 2 +-
...ratorFactory.java => LocalStorageOperator.java} | 27 ++++++++++------------
.../storage/hdfs/LocalStorageOperatorFactory.java | 2 +-
4 files changed, 26 insertions(+), 18 deletions(-)
diff --git
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java
index ab096808b0..945e361a09 100644
---
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java
+++
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java
@@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.plugin.storage.api;
+import static
org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_FILE;
+
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
@@ -72,7 +74,16 @@ public interface StorageOperate {
*/
default String getResourceFileName(String tenantCode, String fullName) {
String resDir = getResDir(tenantCode);
- return fullName.replaceFirst(resDir, "");
+ String filenameReplaceResDir = fullName.replaceFirst(resDir, "");
+ if (!filenameReplaceResDir.equals(fullName)) {
+ return filenameReplaceResDir;
+ }
+
+ // Replace resource dir not effective in case of run workflow with
different tenant from resource file's.
+ // this is backup solution to get related path, by split with
RESOURCE_TYPE_FILE
+ return filenameReplaceResDir.contains(RESOURCE_TYPE_FILE)
+ ? filenameReplaceResDir.split(String.format("%s/",
RESOURCE_TYPE_FILE))[1]
+ : filenameReplaceResDir;
}
/**
diff --git
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java
index f5891a5b92..4479c93a3d 100644
---
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java
+++
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java
@@ -71,7 +71,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
@Slf4j
public class HdfsStorageOperator implements Closeable, StorageOperate {
- private static HdfsStorageProperties hdfsProperties = new
HdfsStorageProperties();
+ protected static HdfsStorageProperties hdfsProperties = new
HdfsStorageProperties();
private static final String HADOOP_UTILS_KEY = "HADOOP_UTILS_KEY";
private volatile boolean yarnEnabled = false;
diff --git
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperator.java
similarity index 53%
copy from
dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java
copy to
dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperator.java
index 1e244b2137..173760b632 100644
---
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java
+++
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperator.java
@@ -17,26 +17,23 @@
package org.apache.dolphinscheduler.plugin.storage.hdfs;
-import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
-import org.apache.dolphinscheduler.plugin.storage.api.StorageOperateFactory;
-import org.apache.dolphinscheduler.plugin.storage.api.StorageType;
+import lombok.extern.slf4j.Slf4j;
-import com.google.auto.service.AutoService;
+@Slf4j
+public class LocalStorageOperator extends HdfsStorageOperator {
-@AutoService(StorageOperateFactory.class)
-public class LocalStorageOperatorFactory implements StorageOperateFactory {
-
- private static final String LOCAL_DEFAULT_FS = "file:/";
+ public LocalStorageOperator() {
+ super(new HdfsStorageProperties());
+ }
- @Override
- public StorageOperate createStorageOperate() {
- HdfsStorageProperties hdfsStorageProperties = new
HdfsStorageProperties();
- hdfsStorageProperties.setDefaultFS(LOCAL_DEFAULT_FS);
- return new HdfsStorageOperator(hdfsStorageProperties);
+ public LocalStorageOperator(HdfsStorageProperties hdfsStorageProperties) {
+ super(hdfsStorageProperties);
}
@Override
- public StorageType getStorageOperate() {
- return StorageType.LOCAL;
+ public String getResourceFileName(String tenantCode, String fullName) {
+ // prefix schema `file:/` should be remove in local file mode
+ String fullNameRemoveSchema =
fullName.replaceFirst(hdfsProperties.getDefaultFS(), "");
+ return super.getResourceFileName(tenantCode, fullNameRemoveSchema);
}
}
diff --git
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java
index 1e244b2137..5f44eca87b 100644
---
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java
+++
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java
@@ -32,7 +32,7 @@ public class LocalStorageOperatorFactory implements
StorageOperateFactory {
public StorageOperate createStorageOperate() {
HdfsStorageProperties hdfsStorageProperties = new
HdfsStorageProperties();
hdfsStorageProperties.setDefaultFS(LOCAL_DEFAULT_FS);
- return new HdfsStorageOperator(hdfsStorageProperties);
+ return new LocalStorageOperator(hdfsStorageProperties);
}
@Override
