This is an automated email from the ASF dual-hosted git repository.
caishunfeng pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 63f835715f [fix] Enhance name pre checker in resource center (#10094)
63f835715f is described below
commit 63f835715f8ca8bff79c0e7177ebfa5917ebb3bd
Author: Jiajie Zhong <[email protected]>
AuthorDate: Wed May 18 18:40:17 2022 +0800
[fix] Enhance name pre checker in resource center (#10094)
* [fix] Enhance name pre checker in resource center
Add file name and directory checker to avoid directory traversal
* add some missing change and change docs
* change var name in directoryTraversal
* Fix ci
---
docs/docs/en/guide/resource/file-manage.md | 31 +++++++++++++---------
docs/docs/zh/guide/resource/file-manage.md | 31 +++++++++++++---------
.../apache/dolphinscheduler/api/enums/Status.java | 2 +-
.../api/service/impl/ResourcesServiceImpl.java | 21 +++++++++++++--
.../dolphinscheduler/common/utils/FileUtils.java | 21 +++++++++++++++
.../common/utils/FileUtilsTest.java | 30 +++++++++++++++++++++
6 files changed, 108 insertions(+), 28 deletions(-)
diff --git a/docs/docs/en/guide/resource/file-manage.md
b/docs/docs/en/guide/resource/file-manage.md
index ae3e8d2ec4..87b965801b 100644
--- a/docs/docs/en/guide/resource/file-manage.md
+++ b/docs/docs/en/guide/resource/file-manage.md
@@ -4,40 +4,47 @@ When third party jars are used in the scheduling process or
user defined scripts
-- Create a file
- > The file format supports the following types: txt, log, sh, conf, cfg, py,
java, sql, xml, hql, properties.
+## Basic Operator
+
+### Create a File
+
+The file format supports the following types: txt, log, sh, conf, cfg, py,
java, sql, xml, hql, properties.
-- upload files
+### Upload Files
-> Upload file: Click the "Upload File" button to upload, drag the file to the
upload area, the file name will be automatically completed with the uploaded
file name.
+Click the "Upload File" button to upload, drag the file to the upload area,
the file name will be automatically completed with the uploaded file name.
-- File View
+### View File Content
-> For the files that can be viewed, click the file name to view the file
details.
+ For the files that can be viewed, click the file name to view the file
details.
-- Download file
+### Download file
> Click the "Download" button in the file list to download the file or click
> the "Download" button in the upper right corner of the file details to
> download the file.
-- File rename
+### Rename File
-- delete
- > File list -> Click the "Delete" button to delete the specified file.
+### Delete File
-- Re-upload file
+File list -> Click the "Delete" button to delete the specified file.
- > Re-upload file: Click the "Re-upload File" button to upload a new file to
replace the old file, drag the file to the re-upload area, the file name will
be automatically completed with the new uploaded file name.
+### Re-upload file
+
+Click the "Re-upload File" button to upload a new file to replace the old
file, drag the file to the re-upload area, the file name will be automatically
completed with the new uploaded file name.
+> Note: File name or source name of your local file can not contain specific
characters like `.` or `/` when you trying to
+> upload, create or rename file in resource center.
+
## Example
The example uses a simple shell script to demonstrate the use of resource
center files in workflow definitions. The same is true for tasks such as MR and
Spark, which require jar packages.
diff --git a/docs/docs/zh/guide/resource/file-manage.md
b/docs/docs/zh/guide/resource/file-manage.md
index 77d9509548..a0d2f0ab56 100644
--- a/docs/docs/zh/guide/resource/file-manage.md
+++ b/docs/docs/zh/guide/resource/file-manage.md
@@ -2,43 +2,48 @@
当在调度过程中需要使用到第三方的 jar
或者用户需要自定义脚本的情况，可以通过在该页面完成相关操作。可创建的文件类型包括：`txt/log/sh/conf/py/java`
等。并且可以对文件进行编辑、重命名、下载和删除等操作。
+## 基础操作
+
-* 创建文件
+### 创建文件
- > 文件格式支持以下几种类型：txt、log、sh、conf、cfg、py、java、sql、xml、hql、properties
+文件格式支持以下几种类型：txt、log、sh、conf、cfg、py、java、sql、xml、hql、properties
-* 上传文件
- > 上传文件：点击"上传文件"按钮进行上传，将文件拖拽到上传区域，文件名会自动以上传的文件名称补全
+### 上传文件
+
+上传文件：点击"上传文件"按钮进行上传，将文件拖拽到上传区域，文件名会自动以上传的文件名称补全
-* 文件查看
+### 文件查看
- > 对可查看的文件类型，点击文件名称，可查看文件详情
+对可查看的文件类型，点击文件名称，可查看文件详情
-* 下载文件
+### 下载文件
- > 点击文件列表的"下载"按钮下载文件或者在文件详情中点击右上角"下载"按钮下载文件
+点击文件列表的"下载"按钮下载文件或者在文件详情中点击右上角"下载"按钮下载文件
-* 文件重命名
+### 文件重命名
-* 删除
+### 删除文件
-> 文件列表->点击"删除"按钮，删除指定文件
+文件列表->点击"删除"按钮，删除指定文件
-* 重新上传文件
+### 重新上传文件
- > 点击文件列表中的”重新上传文件“按钮进行重新上传文件，将文件拖拽到上传区域，文件名会自动以上传的文件名称补全
+点击文件列表中的”重新上传文件“按钮进行重新上传文件，将文件拖拽到上传区域，文件名会自动以上传的文件名称补全
+> 注意：上传、创建、重命名文件时，文件名和源文件名（上传时）均不能带有 `.` 以及 `/` 特殊符号。
+
## 任务样例
该样例主要通过一个简单的 shell 脚本，来演示如何在工作流定义中使用资源中心的文件。像 MR、Spark 等任务需要用到 jar 包，也是同理。
diff --git
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
index e0a38c1e5f..f5313f053b 100644
---
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
+++
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
@@ -396,7 +396,7 @@ public enum Status {
K8S_CLIENT_OPS_ERROR(1300006, "k8s error with exception {0}",
"k8s操作报错[{0}]"),
VERIFY_K8S_NAMESPACE_ERROR(1300007, "verify k8s and namespace error",
"验证k8s命名空间信息错误"),
DELETE_K8S_NAMESPACE_BY_ID_ERROR(1300008, "delete k8s namespace by id
error", "删除命名空间错误"),
- VERIFY_PARAMETER_NAME_FAILED(1300009, "The file name verify failed",
"文件命名校验失败"),
+ VERIFY_PARAMETER_NAME_FAILED(1300009, "The file name verify failed",
"文件命名校验失败"),
STORE_OPERATE_CREATE_ERROR(1300010, "create the resource failed",
"存储操作失败"),
GRANT_K8S_NAMESPACE_ERROR(1300011, "grant namespace error", "授权资源错误"),
QUERY_UNAUTHORIZED_NAMESPACE_ERROR(1300012, "query unauthorized namespace
error", "查询未授权命名空间错误"),
diff --git
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
index 200aeb1c65..1c5804cd81 100644
---
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
+++
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
@@ -129,8 +129,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl
implements ResourcesSe
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
return result;
}
- if (name.endsWith(FOLDER_SEPARATOR)) {
- result.setCode(Status.VERIFY_PARAMETER_NAME_FAILED.getCode());
+ if (FileUtils.directoryTraversal(name)) {
+ putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
return result;
}
String fullName = getFullName(currentDir, name);
@@ -517,6 +517,19 @@ public class ResourcesServiceImpl extends BaseServiceImpl
implements ResourcesSe
private Result<Object> verifyFile(String name, ResourceType type,
MultipartFile file) {
Result<Object> result = new Result<>();
putMsg(result, Status.SUCCESS);
+
+ if (FileUtils.directoryTraversal(name)) {
+ logger.error("file alias name {} verify failed", name);
+ putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
+ return result;
+ }
+
+ if (file != null &&
FileUtils.directoryTraversal(Objects.requireNonNull(file.getOriginalFilename())))
{
+ logger.error("file original name {} verify failed",
file.getOriginalFilename());
+ putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
+ return result;
+ }
+
if (file != null) {
// file is empty
if (file.isEmpty()) {
@@ -973,6 +986,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl
implements ResourcesSe
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
return result;
}
+ if (FileUtils.directoryTraversal(fileName)) {
+ putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
+ return result;
+ }
//check file suffix
String nameSuffix = fileSuffix.trim();
diff --git
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
index e7817ffe80..bdcf62f76a 100644
---
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
+++
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
@@ -204,4 +204,25 @@ public class FileUtils {
}
}
+ /**
+ * Check whether the given string type of path can be traversal or not,
return true if path could
+ * traversal, and return false if it is not.
+ *
+ * @param filename String type of filename
+ * @return whether file path could be traversal or not
+ */
+ public static boolean directoryTraversal(String filename){
+ if (filename.contains(FOLDER_SEPARATOR)) {
+ return true;
+ }
+ File file = new File(filename);
+ try {
+ File canonical = file.getCanonicalFile();
+ File absolute = file.getAbsoluteFile();
+ return !canonical.equals(absolute);
+ } catch (IOException e) {
+ return true;
+ }
+ }
+
}
diff --git
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
index d023b8651f..fdcaccd697 100644
---
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
+++
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
@@ -88,4 +88,34 @@ public class FileUtilsTest {
Assert.assertEquals(content, fileContent);
}
+ @Test
+ public void testDirectoryTraversal() {
+ // test case which do not directory traversal
+ String path;
+ path = "abc.txt";
+ Assert.assertFalse(FileUtils.directoryTraversal(path));
+
+ path = "abc...txt";
+ Assert.assertFalse(FileUtils.directoryTraversal(path));
+
+ path = "..abc.txt";
+ Assert.assertFalse(FileUtils.directoryTraversal(path));
+
+ // test case which will directory traversal
+ path = "../abc.txt";
+ Assert.assertTrue(FileUtils.directoryTraversal(path));
+
+ path = "../../abc.txt";
+ Assert.assertTrue(FileUtils.directoryTraversal(path));
+
+ path = "abc../def.txt";
+ Assert.assertTrue(FileUtils.directoryTraversal(path));
+
+ path = "abc./def.txt";
+ Assert.assertTrue(FileUtils.directoryTraversal(path));
+
+ path = "abc/def...txt";
+ Assert.assertTrue(FileUtils.directoryTraversal(path));
+ }
+
}
