This is an automated email from the ASF dual-hosted git repository.
wenjun pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 7235f25628 [Improvement-17480][Storage] Separate the local storage
implementation from hdfs storage plugin (#17547)
7235f25628 is described below
commit 7235f25628bc1a633a055a5ac36d68bb72af6515
Author: Wenjun Ruan <[email protected]>
AuthorDate: Thu Oct 9 17:23:06 2025 +0800
[Improvement-17480][Storage] Separate the local storage implementation from
hdfs storage plugin (#17547)
---
.../resources/docker/basic/docker-compose.yaml | 10 +-
.../datasource-clickhouse/docker-compose.yaml | 10 +-
.../docker/datasource-hive/docker-compose.yaml | 10 +-
.../docker/datasource-mysql/docker-compose.yaml | 10 +-
.../datasource-postgresql/docker-compose.yaml | 10 +-
.../datasource-sqlserver/docker-compose.yaml | 10 +-
.../docker/file-manage/docker-compose.yaml | 10 +-
.../docker/ldap-login/docker-compose.yaml | 10 +-
dolphinscheduler-dao/pom.xml | 4 -
.../resources/docker/basic/docker-compose.yaml | 11 +-
.../datasource-clickhouse/docker-compose.yaml | 11 +-
.../datasource-dolphindb/docker-compose.yaml | 11 +-
.../docker/datasource-hive/docker-compose.yaml | 11 +-
.../docker/datasource-mysql/docker-compose.yaml | 11 +-
.../datasource-postgresql/docker-compose.yaml | 11 +-
.../datasource-sqlserver/docker-compose.yaml | 11 +-
.../docker/file-manage/docker-compose.yaml | 11 +-
.../docker/python-task/docker-compose.yaml | 11 +-
.../docker/workflow-http/docker-compose.yaml | 18 +-
dolphinscheduler-master/pom.xml | 11 ++
.../storage/api/AbstractStorageOperator.java | 17 +-
.../storage/api/local/LocalStorageOperator.java | 206 +++++++++++++++++++++
.../api/local}/LocalStorageOperatorFactory.java | 18 +-
.../api/local}/LocalStorageOperatorTest.java | 160 ++++++++--------
.../plugin/storage/hdfs/LocalStorageOperator.java | 29 ---
25 files changed, 477 insertions(+), 165 deletions(-)
diff --git
a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/basic/docker-compose.yaml
b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/basic/docker-compose.yaml
index edeca6af11..5438b59e8a 100644
---
a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/basic/docker-compose.yaml
+++
b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/basic/docker-compose.yaml
@@ -21,7 +21,15 @@ services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci
environment:
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
networks:
diff --git
a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml
b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml
index 29e4ba140b..f3047c718f 100644
---
a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml
+++
b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml
@@ -21,7 +21,15 @@ services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci
environment:
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
networks:
diff --git
a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-hive/docker-compose.yaml
b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-hive/docker-compose.yaml
index c8ec0c5792..1cbd859d29 100644
---
a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-hive/docker-compose.yaml
+++
b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-hive/docker-compose.yaml
@@ -21,7 +21,15 @@ services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci
environment:
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
networks:
diff --git
a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml
b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml
index 67e79d8653..5e8bb90f28 100644
---
a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml
+++
b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml
@@ -21,7 +21,15 @@ services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci
environment:
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
networks:
diff --git
a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml
b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml
index 5281556e85..c2bcec2e61 100644
---
a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml
+++
b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml
@@ -21,7 +21,15 @@ services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci
environment:
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
networks:
diff --git
a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml
b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml
index 656f6a3ecc..eb9b8e5f6b 100644
---
a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml
+++
b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml
@@ -21,7 +21,15 @@ services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci
environment:
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
networks:
diff --git
a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/docker-compose.yaml
b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/docker-compose.yaml
index dbe25bcbad..df7257a6b2 100644
---
a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/docker-compose.yaml
+++
b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/docker-compose.yaml
@@ -21,7 +21,15 @@ services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci
environment:
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
networks:
diff --git
a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/ldap-login/docker-compose.yaml
b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/ldap-login/docker-compose.yaml
index d22b8725e4..4b76e114e7 100644
---
a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/ldap-login/docker-compose.yaml
+++
b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/ldap-login/docker-compose.yaml
@@ -21,7 +21,15 @@ services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci
environment:
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
volumes:
diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml
index f46817c0cc..720c1aa51c 100644
--- a/dolphinscheduler-dao/pom.xml
+++ b/dolphinscheduler-dao/pom.xml
@@ -103,10 +103,6 @@
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-storage-api</artifactId>
- </dependency>
</dependencies>
<build>
diff --git
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml
index 37bebedc64..d124cd90e2 100644
---
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml
+++
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml
@@ -21,8 +21,15 @@ services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci
environment:
- MASTER_MAX_CPU_LOAD_AVG: 100
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
networks:
diff --git
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml
index ec108f067e..2cf3743c5e 100644
---
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml
+++
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml
@@ -21,8 +21,15 @@ services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci
environment:
- MASTER_MAX_CPU_LOAD_AVG: 100
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
networks:
diff --git
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-dolphindb/docker-compose.yaml
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-dolphindb/docker-compose.yaml
index b1831dabf6..413c67e3db 100644
---
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-dolphindb/docker-compose.yaml
+++
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-dolphindb/docker-compose.yaml
@@ -21,8 +21,15 @@ services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci
environment:
- MASTER_MAX_CPU_LOAD_AVG: 100
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
networks:
diff --git
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-hive/docker-compose.yaml
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-hive/docker-compose.yaml
index d28abc8c22..9c36411a11 100644
---
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-hive/docker-compose.yaml
+++
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-hive/docker-compose.yaml
@@ -21,8 +21,15 @@ services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci
environment:
- MASTER_MAX_CPU_LOAD_AVG: 100
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
networks:
diff --git
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml
index 7a35995310..a1d2b1c72f 100644
---
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml
+++
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml
@@ -21,8 +21,15 @@ services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci
environment:
- MASTER_MAX_CPU_LOAD_AVG: 100
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
networks:
diff --git
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml
index 878bf24364..6d851e195e 100644
---
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml
+++
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml
@@ -21,8 +21,15 @@ services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci
environment:
- MASTER_MAX_CPU_LOAD_AVG: 100
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
networks:
diff --git
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml
index 26e76d26b6..b91073845d 100644
---
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml
+++
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml
@@ -21,8 +21,15 @@ services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci
environment:
- MASTER_MAX_CPU_LOAD_AVG: 100
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
networks:
diff --git
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml
index 46d24d55e9..81f9c98559 100644
---
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml
+++
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml
@@ -21,8 +21,15 @@ services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci
environment:
- MASTER_MAX_CPU_LOAD_AVG: 100
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
networks:
diff --git
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/python-task/docker-compose.yaml
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/python-task/docker-compose.yaml
index 3a287cce3d..d66469dedf 100644
---
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/python-task/docker-compose.yaml
+++
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/python-task/docker-compose.yaml
@@ -24,8 +24,15 @@ services:
context: .
dockerfile: ./Dockerfile
environment:
- MASTER_MAX_CPU_LOAD_AVG: 100
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
networks:
diff --git
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/workflow-http/docker-compose.yaml
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/workflow-http/docker-compose.yaml
index 0f6a4a6205..3d766e4223 100644
---
a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/workflow-http/docker-compose.yaml
+++
b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/workflow-http/docker-compose.yaml
@@ -21,9 +21,15 @@ services:
dolphinscheduler:
image: apache/dolphinscheduler-standalone-server:ci
environment:
- MASTER_MAX_CPU_LOAD_AVG: 400
- MASTER_RESERVED_MEMORY: 0.01
- WORKER_TENANT_AUTO_CREATE: 'true'
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ MASTER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
MASTER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_SERVER_LOAD_PROTECTION_MAX_JVM_CPU_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS:
0.95
+
WORKER_SERVER_LOAD_PROTECTION_MAX_DISK_MEMORY_USAGE_PERCENTAGE_THRESHOLDS: 0.95
+ WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: 'true'
ports:
- "12345:12345"
networks:
@@ -42,9 +48,9 @@ services:
networks:
- e2e
volumes:
- - type: bind
- source: ./
- target: /config
+ - type: bind
+ source: ./
+ target: /config
healthcheck:
test: [ "CMD", "curl", "http://localhost:1080/"; ]
interval: 5s
diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml
index 28fb4c3f61..71883ad46a 100644
--- a/dolphinscheduler-master/pom.xml
+++ b/dolphinscheduler-master/pom.xml
@@ -94,6 +94,17 @@
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-storage-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-storage-all</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-extract-master</artifactId>
diff --git
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java
index 76a0f3096f..af27d164af 100644
---
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java
+++
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java
@@ -18,23 +18,17 @@
package org.apache.dolphinscheduler.plugin.storage.api;
import org.apache.dolphinscheduler.common.utils.FileUtils;
-import org.apache.dolphinscheduler.common.utils.PropertyUtils;
-import
org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.lang3.StringUtils;
import java.io.File;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import com.google.common.base.Preconditions;
import com.google.common.io.Files;
public abstract class AbstractStorageOperator implements StorageOperator {
- private static final Logger log =
LoggerFactory.getLogger(AbstractStorageOperator.class);
protected final String resourceBaseAbsolutePath;
public AbstractStorageOperator(String resourceBaseAbsolutePath) {
@@ -64,7 +58,7 @@ public abstract class AbstractStorageOperator implements
StorageOperator {
@Override
public String getStorageBaseDirectory() {
// All directory should end with File.separator
- return PropertyUtils.getString(StorageConstants.RESOURCE_UPLOAD_PATH,
"/tmp/dolphinscheduler");
+ return resourceBaseAbsolutePath;
}
@Override
@@ -108,4 +102,13 @@ public abstract class AbstractStorageOperator implements
StorageOperator {
}
}
+ protected void exceptionIfPathNotUnderStorageBaseDir(String
resourceAbsolutePath) {
+ String storageBaseDirectory = getStorageBaseDirectory();
+ if (!resourceAbsolutePath.startsWith(storageBaseDirectory)) {
+ throw new IllegalArgumentException(
+ "Resource path: " + resourceAbsolutePath + " is not under
storage base directory: "
+ + storageBaseDirectory);
+ }
+ }
+
}
diff --git
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/local/LocalStorageOperator.java
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/local/LocalStorageOperator.java
new file mode 100644
index 0000000000..a0a8190e09
--- /dev/null
+++
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/local/LocalStorageOperator.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.storage.api.local;
+
+import org.apache.dolphinscheduler.plugin.storage.api.AbstractStorageOperator;
+import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
+
+import org.apache.commons.io.FileUtils;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+import com.google.common.collect.Lists;
+
+@Slf4j
+public class LocalStorageOperator extends AbstractStorageOperator implements
Closeable, StorageOperator {
+
+ public LocalStorageOperator(String resourceBaseAbsolutePath) throws
IOException {
+ super(resourceBaseAbsolutePath);
+ final Path path = Paths.get(resourceBaseAbsolutePath);
+ if (Files.exists(path)) {
+ if (!Files.isDirectory(path)) {
+ throw new IllegalArgumentException("The base path must be a
directory: " + resourceBaseAbsolutePath);
+ }
+ } else {
+ Files.createDirectories(path);
+ }
+ }
+
+ @SneakyThrows
+ @Override
+ public void createStorageDir(String directoryAbsolutePath) {
+ final Path path = Paths.get(directoryAbsolutePath);
+ if (exists(directoryAbsolutePath)) {
+ throw new FileAlreadyExistsException("Directory already exists: "
+ directoryAbsolutePath);
+ }
+ Files.createDirectories(path);
+ }
+
+ @Override
+ public boolean exists(String resourceAbsolutePath) {
+ return Files.exists(Paths.get(resourceAbsolutePath));
+ }
+
+ @SneakyThrows
+ @Override
+ public void delete(String resourceAbsolutePath, boolean recursive) {
+ if (recursive) {
+ FileUtils.deleteQuietly(new File(resourceAbsolutePath));
+ } else {
+ Files.deleteIfExists(Paths.get(resourceAbsolutePath));
+ }
+ }
+
+ @SneakyThrows
+ @Override
+ public void copy(String srcAbsolutePath, String dstAbsolutePath, boolean
deleteSource, boolean overwrite) {
+ if (srcAbsolutePath.equals(dstAbsolutePath)) {
+ throw new IllegalArgumentException(
+ "Source path and destination path cannot be the same: " +
srcAbsolutePath);
+ }
+
+ if (!exists(srcAbsolutePath)) {
+ throw new FileNotFoundException("Source path does not exist: " +
srcAbsolutePath);
+ }
+
+ if (exists(dstAbsolutePath)) {
+ if (!overwrite) {
+ throw new FileAlreadyExistsException("Destination path already
exists: " + dstAbsolutePath);
+ }
+ delete(dstAbsolutePath, true);
+ }
+
+ final File srcFile = new File(srcAbsolutePath);
+ final File dstFile = new File(dstAbsolutePath);
+ if (FileUtils.isDirectory(srcFile)) {
+ FileUtils.copyDirectoryToDirectory(srcFile, dstFile);
+ } else {
+ FileUtils.copyFile(srcFile, dstFile);
+ }
+ if (deleteSource) {
+ delete(srcAbsolutePath, true);
+ }
+ }
+
+ @Override
+ public void upload(String srcLocalFileAbsolutePath, String
dstAbsolutePath, boolean deleteSource,
+ boolean overwrite) {
+ copy(srcLocalFileAbsolutePath, dstAbsolutePath, deleteSource,
overwrite);
+ }
+
+ @Override
+ public void download(String srcFileAbsolutePath, String dstAbsolutePath,
boolean overwrite) {
+ copy(srcFileAbsolutePath, dstAbsolutePath, false, overwrite);
+ }
+
+ @SneakyThrows
+ @Override
+ public List<String> fetchFileContent(String fileAbsolutePath, int
skipLineNums, int limit) {
+ try (Stream<String> stream =
Files.lines(Paths.get(fileAbsolutePath)).skip(skipLineNums).limit(limit)) {
+ return stream.collect(Collectors.toList());
+ }
+ }
+
+ @SneakyThrows
+ @Override
+ public List<StorageEntity> listStorageEntity(String resourceAbsolutePath) {
+ Path path = Paths.get(resourceAbsolutePath);
+ if (!Files.exists(path)) {
+ return Collections.emptyList();
+ }
+ if (Files.isDirectory(path)) {
+ try (Stream<Path> stream = Files.list(path)) {
+ return
stream.map(this::transformFileStatusToResourceMetadata).collect(Collectors.toList());
+ }
+ }
+ return Lists.newArrayList(transformFileStatusToResourceMetadata(path));
+ }
+
+ @SneakyThrows
+ @Override
+ public List<StorageEntity> listFileStorageEntityRecursively(String
resourceAbsolutePath) {
+ List<StorageEntity> result = new ArrayList<>();
+
+ LinkedList<Path> foldersToFetch = new LinkedList<>();
+ foldersToFetch.addLast(Paths.get(resourceAbsolutePath));
+
+ while (!foldersToFetch.isEmpty()) {
+ final Path path1 = foldersToFetch.pollFirst();
+ if (!Files.exists(path1)) {
+ continue;
+ }
+ try (Stream<Path> children = Files.list(path1)) {
+ children.forEach(child -> {
+ if (child.toFile().isDirectory()) {
+ foldersToFetch.add(child);
+ }
+ result.add(transformFileStatusToResourceMetadata(child));
+ });
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public StorageEntity getStorageEntity(String resourceAbsolutePath) {
+ return
transformFileStatusToResourceMetadata(Paths.get(resourceAbsolutePath));
+ }
+
+ @Override
+ public void close() {
+ // ignore
+ }
+
+ @SneakyThrows
+ private StorageEntity transformFileStatusToResourceMetadata(Path path) {
+ final BasicFileAttributes attrs = Files.readAttributes(path,
BasicFileAttributes.class);
+ final String fileAbsolutePath = path.toAbsolutePath().toString();
+ final ResourceMetadata resourceMetaData =
getResourceMetaData(fileAbsolutePath);
+ return StorageEntity.builder()
+ .fileName(path.getFileName().toString())
+ .fullName(fileAbsolutePath)
+ .pfullName(resourceMetaData.getResourceParentAbsolutePath())
+ .type(resourceMetaData.getResourceType())
+ .isDirectory(attrs.isDirectory())
+ .size(attrs.size())
+ .relativePath(resourceMetaData.getResourceRelativePath())
+ .createTime(new Date(attrs.creationTime().toMillis()))
+ .updateTime(new Date(attrs.lastModifiedTime().toMillis()))
+ .build();
+ }
+}
diff --git
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/local/LocalStorageOperatorFactory.java
similarity index 72%
rename from
dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java
rename to
dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/local/LocalStorageOperatorFactory.java
index 0038342265..243d70f68c 100644
---
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java
+++
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/local/LocalStorageOperatorFactory.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.plugin.storage.hdfs;
+package org.apache.dolphinscheduler.plugin.storage.api.local;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
@@ -23,24 +23,18 @@ import
org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory;
import org.apache.dolphinscheduler.plugin.storage.api.StorageType;
import
org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants;
+import lombok.SneakyThrows;
+
import com.google.auto.service.AutoService;
@AutoService(StorageOperatorFactory.class)
public class LocalStorageOperatorFactory implements StorageOperatorFactory {
- public static final String LOCAL_DEFAULT_FS = "file:/";
-
+ @SneakyThrows
@Override
public StorageOperator createStorageOperate() {
- final HdfsStorageProperties hdfsStorageProperties =
getHdfsStorageProperties();
- return new LocalStorageOperator(hdfsStorageProperties);
- }
-
- private HdfsStorageProperties getHdfsStorageProperties() {
- return HdfsStorageProperties.builder()
- .defaultFS(LOCAL_DEFAULT_FS)
-
.resourceUploadPath(PropertyUtils.getString(StorageConstants.RESOURCE_UPLOAD_PATH,
"/dolphinscheduler"))
- .build();
+ return new LocalStorageOperator(
+ PropertyUtils.getString(StorageConstants.RESOURCE_UPLOAD_PATH,
"/tmp/dolphinscheduler"));
}
@Override
diff --git
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/test/java/org/apache/dolphinscheduler/plugin/storage/api/local/LocalStorageOperatorTest.java
similarity index 62%
rename from
dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java
rename to
dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/test/java/org/apache/dolphinscheduler/plugin/storage/api/local/LocalStorageOperatorTest.java
index 899f17873a..5a00890ae5 100644
---
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java
+++
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/test/java/org/apache/dolphinscheduler/plugin/storage/api/local/LocalStorageOperatorTest.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.plugin.storage.hdfs;
+package org.apache.dolphinscheduler.plugin.storage.api.local;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -42,56 +42,56 @@ class LocalStorageOperatorTest {
private StorageOperator storageOperator;
- private static final String resourceBaseDir =
+ private static final String STORAGE_BASE_DIR =
Paths.get(LocalStorageOperatorTest.class.getResource("/").getFile(),
"localStorage").toString();
- private static final String tenantCode = "default";
- private static final String baseDir =
- Paths.get(resourceBaseDir, tenantCode,
StorageOperator.FILE_FOLDER_NAME).toString();
+ private static final String TENANT_CODE = "default";
+ private static final String TENANT_BASE_DIR =
+ Paths.get(STORAGE_BASE_DIR, TENANT_CODE,
StorageOperator.FILE_FOLDER_NAME).toString();
@SneakyThrows
@BeforeEach
- public void setup() {
- Files.createDirectories(Paths.get(resourceBaseDir));
+ void setup() {
+ // /localStorage/default/resources/sqlDirectory/demo.sql
+ // /emptyDirectory
+ Files.createDirectories(Paths.get(STORAGE_BASE_DIR));
System.clearProperty(StorageConstants.RESOURCE_UPLOAD_PATH);
- System.setProperty(StorageConstants.RESOURCE_UPLOAD_PATH,
resourceBaseDir);
+ System.setProperty(StorageConstants.RESOURCE_UPLOAD_PATH,
STORAGE_BASE_DIR);
LocalStorageOperatorFactory localStorageOperatorFactory = new
LocalStorageOperatorFactory();
storageOperator = localStorageOperatorFactory.createStorageOperate();
// create file and directory
- Files.createDirectories(Paths.get(baseDir, "sqlDirectory"));
- Files.createDirectories(Paths.get(baseDir, "emptyDirectory"));
- Files.createFile(Paths.get(baseDir, "sqlDirectory", "demo.sql"));
- Files.write(Paths.get(baseDir, "sqlDirectory", "demo.sql"), "select *
from demo".getBytes());
+ Files.createDirectories(Paths.get(TENANT_BASE_DIR, "sqlDirectory"));
+ Files.createDirectories(Paths.get(TENANT_BASE_DIR, "emptyDirectory"));
+ Files.createFile(Paths.get(TENANT_BASE_DIR, "sqlDirectory",
"demo.sql"));
+ Files.write(Paths.get(TENANT_BASE_DIR, "sqlDirectory", "demo.sql"),
"select * from demo".getBytes());
}
@Test
- public void testGetResourceMetaData_directory() {
- String resourceFileAbsolutePath = "file:" + baseDir;
-
- ResourceMetadata resourceMetaData =
storageOperator.getResourceMetaData(resourceFileAbsolutePath);
-
assertThat(resourceMetaData.getResourceAbsolutePath()).isEqualTo("file:" +
baseDir);
-
assertThat(resourceMetaData.getResourceBaseDirectory()).isEqualTo("file:" +
resourceBaseDir);
+ void testGetResourceMetaData_directory() {
+ ResourceMetadata resourceMetaData =
storageOperator.getResourceMetaData(TENANT_BASE_DIR);
+
assertThat(resourceMetaData.getResourceAbsolutePath()).isEqualTo(TENANT_BASE_DIR);
+
assertThat(resourceMetaData.getResourceBaseDirectory()).isEqualTo(STORAGE_BASE_DIR);
assertThat(resourceMetaData.getTenant()).isEqualTo("default");
assertThat(resourceMetaData.getResourceType()).isEqualTo(ResourceType.FILE);
assertThat(resourceMetaData.getResourceRelativePath()).isEqualTo("/");
}
@Test
- public void testGetResourceMetaData_file() {
- String resourceFileAbsolutePath = "file:" + Paths.get(baseDir,
"sqlDirectory", "demo.sql");
+ void testGetResourceMetaData_file() {
+ String resourceFileAbsolutePath = Paths.get(TENANT_BASE_DIR,
"sqlDirectory", "demo.sql").toString();
ResourceMetadata resourceMetaData =
storageOperator.getResourceMetaData(resourceFileAbsolutePath);
assertThat(resourceMetaData.getResourceAbsolutePath()).isEqualTo(resourceFileAbsolutePath);
-
assertThat(resourceMetaData.getResourceBaseDirectory()).isEqualTo("file:" +
resourceBaseDir);
+
assertThat(resourceMetaData.getResourceBaseDirectory()).isEqualTo(STORAGE_BASE_DIR);
assertThat(resourceMetaData.getTenant()).isEqualTo("default");
assertThat(resourceMetaData.getResourceType()).isEqualTo(ResourceType.FILE);
assertThat(resourceMetaData.getResourceRelativePath()).isEqualTo("sqlDirectory/demo.sql");
}
@Test
- public void testGetResourceMetaData_invalidatedPath() {
- String resourceFileAbsolutePath = Paths.get(baseDir, "sqlDirectory",
"demo.sql").toString();
+ void testGetResourceMetaData_invalidatedPath() {
+ String resourceFileAbsolutePath = Paths.get("/", "sqlDirectory",
"demo.sql").toString();
IllegalArgumentException illegalArgumentException =
assertThrows(IllegalArgumentException.class,
() ->
storageOperator.getResourceMetaData(resourceFileAbsolutePath));
@@ -100,63 +100,63 @@ class LocalStorageOperatorTest {
}
@Test
- public void testGetStorageBaseDirectory() {
+ void testGetStorageBaseDirectory() {
String storageBaseDirectory =
storageOperator.getStorageBaseDirectory();
- assertThat(storageBaseDirectory).isEqualTo("file:" + resourceBaseDir);
+ assertThat(storageBaseDirectory).isEqualTo(STORAGE_BASE_DIR);
}
@Test
- public void testGetStorageBaseDirectory_withTenant() {
+ void testGetStorageBaseDirectory_withTenant() {
String storageBaseDirectory =
storageOperator.getStorageBaseDirectory("default");
- assertThat(storageBaseDirectory).isEqualTo("file:" +
Paths.get(resourceBaseDir, tenantCode));
+ assertThat(storageBaseDirectory).isEqualTo(Paths.get(STORAGE_BASE_DIR,
TENANT_CODE).toString());
}
@Test
- public void testGetStorageBaseDirectory_withTenant_withResourceTypeFile() {
+ void testGetStorageBaseDirectory_withTenant_withResourceTypeFile() {
String storageBaseDirectory =
storageOperator.getStorageBaseDirectory("default", ResourceType.FILE);
assertThat(storageBaseDirectory)
- .isEqualTo("file:" + Paths.get(resourceBaseDir, tenantCode,
StorageOperator.FILE_FOLDER_NAME));
+ .isEqualTo(Paths.get(STORAGE_BASE_DIR, TENANT_CODE,
StorageOperator.FILE_FOLDER_NAME).toString());
}
@Test
- public void testGetStorageBaseDirectory_withTenant_withResourceTypeAll() {
+ void testGetStorageBaseDirectory_withTenant_withResourceTypeAll() {
String storageBaseDirectory =
storageOperator.getStorageBaseDirectory("default", ResourceType.ALL);
- assertThat(storageBaseDirectory).isEqualTo("file:" +
Paths.get(resourceBaseDir, tenantCode));
+ assertThat(storageBaseDirectory).isEqualTo(Paths.get(STORAGE_BASE_DIR,
TENANT_CODE).toString());
}
@Test
- public void testGetStorageBaseDirectory_withEmptyTenant_withResourceType()
{
+ void testGetStorageBaseDirectory_withEmptyTenant_withResourceType() {
IllegalArgumentException illegalArgumentException =
assertThrows(IllegalArgumentException.class,
() -> storageOperator.getStorageBaseDirectory("",
ResourceType.ALL));
assertThat(illegalArgumentException.getMessage()).isEqualTo("Tenant
code should not be empty");
}
@Test
- public void testGetStorageBaseDirectory_withTenant_withEmptyResourceType()
{
+ void testGetStorageBaseDirectory_withTenant_withEmptyResourceType() {
IllegalArgumentException illegalArgumentException =
assertThrows(IllegalArgumentException.class,
() -> storageOperator.getStorageBaseDirectory("default",
null));
assertThat(illegalArgumentException.getMessage()).isEqualTo("Resource
type should not be null");
}
@Test
- public void testGetStorageFileAbsolutePath() {
+ void testGetStorageFileAbsolutePath() {
String fileAbsolutePath =
storageOperator.getStorageFileAbsolutePath("default", "test.sh");
assertThat(fileAbsolutePath).isEqualTo(
- "file:" + Paths.get(resourceBaseDir, tenantCode,
StorageOperator.FILE_FOLDER_NAME, "test.sh"));
+ Paths.get(STORAGE_BASE_DIR, TENANT_CODE,
StorageOperator.FILE_FOLDER_NAME, "test.sh").toString());
}
@SneakyThrows
@Test
- public void testCreateStorageDir_notExists() {
+ void testCreateStorageDir_notExists() {
String testDirFileAbsolutePath =
- "file:" + Paths.get(resourceBaseDir, "root",
StorageOperator.FILE_FOLDER_NAME, "testDir");
+ Paths.get(STORAGE_BASE_DIR, "root",
StorageOperator.FILE_FOLDER_NAME, "testDir").toString();
try {
storageOperator.createStorageDir(testDirFileAbsolutePath);
StorageEntity storageEntity =
storageOperator.getStorageEntity(testDirFileAbsolutePath);
assertThat(storageEntity.getFullName()).isEqualTo(testDirFileAbsolutePath);
assertThat(storageEntity.getFileName()).isEqualTo("testDir");
assertThat(storageEntity.getPfullName())
- .isEqualTo("file:" + Paths.get(resourceBaseDir, "root",
StorageOperator.FILE_FOLDER_NAME));
+ .isEqualTo(Paths.get(STORAGE_BASE_DIR, "root",
StorageOperator.FILE_FOLDER_NAME).toString());
assertThat(storageEntity.isDirectory()).isTrue();
assertThat(storageEntity.getType()).isEqualTo(ResourceType.FILE);
} finally {
@@ -166,39 +166,39 @@ class LocalStorageOperatorTest {
@SneakyThrows
@Test
- public void testCreateStorageDir_exists() {
+ void testCreateStorageDir_exists() {
String testDirFileAbsolutePath =
- "file:" + Paths.get(resourceBaseDir, "default",
StorageOperator.FILE_FOLDER_NAME, "sqlDirectory");
+ Paths.get(STORAGE_BASE_DIR, "default",
StorageOperator.FILE_FOLDER_NAME, "sqlDirectory").toString();
assertThrows(FileAlreadyExistsException.class, () ->
storageOperator.createStorageDir(testDirFileAbsolutePath));
}
@Test
- public void testExists_fileExist() {
- String resourceFileAbsolutePath = "file:" + Paths.get(baseDir,
"sqlDirectory", "demo.sql");
+ void testExists_fileExist() {
+ String resourceFileAbsolutePath = Paths.get(TENANT_BASE_DIR,
"sqlDirectory", "demo.sql").toString();
assertThat(storageOperator.exists(resourceFileAbsolutePath)).isTrue();
}
@Test
- public void testExists_fileNotExist() {
- String resourceFileAbsolutePath = "file:" + Paths.get(baseDir,
"sqlDirectory", "demo.sh");
+ void testExists_fileNotExist() {
+ String resourceFileAbsolutePath = Paths.get(TENANT_BASE_DIR,
"sqlDirectory", "demo.sh").toString();
assertThat(storageOperator.exists(resourceFileAbsolutePath)).isFalse();
}
@Test
- public void testExists_directoryExist() {
- String resourceFileAbsolutePath = "file:" + Paths.get(baseDir,
"sqlDirectory");
+ void testExists_directoryExist() {
+ String resourceFileAbsolutePath = Paths.get(TENANT_BASE_DIR,
"sqlDirectory").toString();
assertThat(storageOperator.exists(resourceFileAbsolutePath)).isTrue();
}
@Test
- public void testExists_directoryNotExist() {
- String resourceFileAbsolutePath = "file:" + Paths.get(baseDir,
"shellDirectory");
+ void testExists_directoryNotExist() {
+ String resourceFileAbsolutePath = Paths.get(TENANT_BASE_DIR,
"shellDirectory").toString();
assertThat(storageOperator.exists(resourceFileAbsolutePath)).isFalse();
}
@Test
- public void testDelete_directoryExist() {
- String resourceFileAbsolutePath = "file:" + Paths.get(baseDir,
"sqlDirectory");
+ void testDelete_directoryExist() {
+ String resourceFileAbsolutePath = Paths.get(TENANT_BASE_DIR,
"sqlDirectory").toString();
assertThat(storageOperator.exists(resourceFileAbsolutePath)).isTrue();
storageOperator.delete(resourceFileAbsolutePath, true);
@@ -206,8 +206,8 @@ class LocalStorageOperatorTest {
}
@Test
- public void testDelete_directoryNotExist() {
- String resourceFileAbsolutePath = "file:" + Paths.get(baseDir,
"shellDirectory");
+ void testDelete_directoryNotExist() {
+ String resourceFileAbsolutePath = Paths.get(TENANT_BASE_DIR,
"shellDirectory").toString();
assertThat(storageOperator.exists(resourceFileAbsolutePath)).isFalse();
storageOperator.delete(resourceFileAbsolutePath, true);
@@ -215,8 +215,8 @@ class LocalStorageOperatorTest {
}
@Test
- public void testDelete_fileExist() {
- String resourceFileAbsolutePath = "file:" + Paths.get(baseDir,
"sqlDirectory", "demo.sql");
+ void testDelete_fileExist() {
+ String resourceFileAbsolutePath = Paths.get(TENANT_BASE_DIR,
"sqlDirectory", "demo.sql").toString();
assertThat(storageOperator.exists(resourceFileAbsolutePath)).isTrue();
storageOperator.delete(resourceFileAbsolutePath, true);
@@ -224,8 +224,8 @@ class LocalStorageOperatorTest {
}
@Test
- public void testDelete_fileNotExist() {
- String resourceFileAbsolutePath = "file:" + Paths.get(baseDir,
"sqlDirectory", "demo.sh");
+ void testDelete_fileNotExist() {
+ String resourceFileAbsolutePath = Paths.get(TENANT_BASE_DIR,
"sqlDirectory", "demo.sh").toString();
assertThat(storageOperator.exists(resourceFileAbsolutePath)).isFalse();
storageOperator.delete(resourceFileAbsolutePath, true);
@@ -233,68 +233,66 @@ class LocalStorageOperatorTest {
}
@Test
- public void testFetchFileContent() {
+ void testFetchFileContent() {
// todo: add large file test case
- String resourceFileAbsolutePath = "file:" + Paths.get(baseDir,
"sqlDirectory", "demo.sql");
+ String resourceFileAbsolutePath = Paths.get(TENANT_BASE_DIR,
"sqlDirectory", "demo.sql").toString();
List<String> content =
storageOperator.fetchFileContent(resourceFileAbsolutePath, 0, 10);
assertThat(content).containsExactly("select * from demo");
}
@Test
- public void testListStorageEntity_directoryNotEmpty() {
- String resourceFileAbsolutePath = "file:" + baseDir;
- List<StorageEntity> storageEntities =
storageOperator.listStorageEntity(resourceFileAbsolutePath);
+ void testListStorageEntity_directoryNotEmpty() {
+ List<StorageEntity> storageEntities =
storageOperator.listStorageEntity(TENANT_BASE_DIR);
assertThat(storageEntities.size()).isEqualTo(2);
StorageEntity storageEntity1 = storageEntities.get(0);
- assertThat(storageEntity1.getFullName()).isEqualTo("file:" + baseDir +
"/emptyDirectory");
+ assertThat(storageEntity1.getFullName()).isEqualTo(TENANT_BASE_DIR +
"/emptyDirectory");
assertThat(storageEntity1.getFileName()).isEqualTo("emptyDirectory");
- assertThat(storageEntity1.getPfullName()).isEqualTo("file:" + baseDir);
+ assertThat(storageEntity1.getPfullName()).isEqualTo(TENANT_BASE_DIR);
assertThat(storageEntity1.isDirectory()).isTrue();
assertThat(storageEntity1.getType()).isEqualTo(ResourceType.FILE);
StorageEntity storageEntity2 = storageEntities.get(1);
- assertThat(storageEntity2.getFullName()).isEqualTo("file:" + baseDir +
"/sqlDirectory");
+ assertThat(storageEntity2.getFullName()).isEqualTo(TENANT_BASE_DIR +
"/sqlDirectory");
assertThat(storageEntity2.getFileName()).isEqualTo("sqlDirectory");
- assertThat(storageEntity2.getPfullName()).isEqualTo("file:" + baseDir);
+ assertThat(storageEntity2.getPfullName()).isEqualTo(TENANT_BASE_DIR);
assertThat(storageEntity2.isDirectory()).isTrue();
assertThat(storageEntity2.getType()).isEqualTo(ResourceType.FILE);
}
@Test
- public void testListStorageEntity_directoryEmpty() {
- String resourceFileAbsolutePath = "file:" + baseDir +
"/emptyDirectory";
+ void testListStorageEntity_directoryEmpty() {
+ String resourceFileAbsolutePath = TENANT_BASE_DIR + "/emptyDirectory";
List<StorageEntity> storageEntities =
storageOperator.listStorageEntity(resourceFileAbsolutePath);
assertThat(storageEntities.size()).isEqualTo(0);
}
@Test
- public void testListStorageEntity_directoryNotExist() {
- String resourceFileAbsolutePath = "file:" + baseDir +
"/notExistDirectory";
+ void testListStorageEntity_directoryNotExist() {
+ String resourceFileAbsolutePath = TENANT_BASE_DIR +
"/notExistDirectory";
assertThat(storageOperator.listStorageEntity(resourceFileAbsolutePath)).isEmpty();
}
@Test
- public void testListStorageEntity_file() {
- String resourceFileAbsolutePath = "file:" + Paths.get(baseDir,
"sqlDirectory", "demo.sql");
+ void testListStorageEntity_file() {
+ String resourceFileAbsolutePath = Paths.get(TENANT_BASE_DIR,
"sqlDirectory", "demo.sql").toString();
List<StorageEntity> storageEntities =
storageOperator.listStorageEntity(resourceFileAbsolutePath);
assertThat(storageEntities.size()).isEqualTo(1);
StorageEntity storageEntity = storageEntities.get(0);
- assertThat(storageEntity.getFullName()).isEqualTo("file:" +
Paths.get(baseDir, "sqlDirectory", "demo.sql"));
+ assertThat(storageEntity.getFullName())
+ .isEqualTo(Paths.get(TENANT_BASE_DIR, "sqlDirectory",
"demo.sql").toString());
assertThat(storageEntity.getFileName()).isEqualTo("demo.sql");
- assertThat(storageEntity.getPfullName()).isEqualTo("file:" +
Paths.get(baseDir, "sqlDirectory"));
+
assertThat(storageEntity.getPfullName()).isEqualTo(Paths.get(TENANT_BASE_DIR,
"sqlDirectory").toString());
assertThat(storageEntity.isDirectory()).isFalse();
assertThat(storageEntity.getType()).isEqualTo(ResourceType.FILE);
}
@Test
- public void testListStorageEntityRecursively_directory() {
- String resourceFileAbsolutePath = "file:" + baseDir;
- List<StorageEntity> storageEntities =
-
storageOperator.listFileStorageEntityRecursively(resourceFileAbsolutePath);
+ void testListStorageEntityRecursively_directory() {
+ List<StorageEntity> storageEntities =
storageOperator.listFileStorageEntityRecursively(TENANT_BASE_DIR);
assertThat(storageEntities.size()).isEqualTo(3);
StorageEntity storageEntity2 = storageEntities.stream()
@@ -302,17 +300,17 @@ class LocalStorageOperatorTest {
.findFirst()
.get();
assertThat(storageEntity2.getFullName())
- .isEqualTo("file:" + Paths.get(baseDir, "sqlDirectory",
"demo.sql"));
+ .isEqualTo(Paths.get(TENANT_BASE_DIR, "sqlDirectory",
"demo.sql").toString());
assertThat(storageEntity2.getFileName()).isEqualTo("demo.sql");
- assertThat(storageEntity2.getPfullName()).isEqualTo("file:" +
Paths.get(baseDir, "sqlDirectory"));
+
assertThat(storageEntity2.getPfullName()).isEqualTo(Paths.get(TENANT_BASE_DIR,
"sqlDirectory").toString());
assertThat(storageEntity2.isDirectory()).isFalse();
assertThat(storageEntity2.getType()).isEqualTo(ResourceType.FILE);
}
@SneakyThrows
@AfterEach
- public void after() {
- FileUtils.deleteFile(resourceBaseDir);
+ void after() {
+ FileUtils.deleteFile(STORAGE_BASE_DIR);
}
}
diff --git
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperator.java
b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperator.java
deleted file mode 100644
index 31c8da5074..0000000000
---
a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperator.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.storage.hdfs;
-
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class LocalStorageOperator extends HdfsStorageOperator {
-
- public LocalStorageOperator(HdfsStorageProperties hdfsStorageProperties) {
- super(hdfsStorageProperties);
- }
-
-}
