This is an automated email from the ASF dual-hosted git repository.
tianqiyan pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new d324cce6f1 [Feature-12526][Api] add dynamic task function: shell task
(#12840)
d324cce6f1 is described below
commit d324cce6f1432c8c2e5669d6a9c151691bf44dfb
Author: Tq <[email protected]>
AuthorDate: Mon Nov 14 13:52:44 2022 +0800
[Feature-12526][Api] add dynamic task function: shell task (#12840)
* submit dynamic task code
* submit shell.json
* add dynamic-task-type-config.yaml to standalone server
---
.../DynamicTaskTypeConfiguration.java | 94 ++++++++++++++++++++
.../api/controller/DynamicTaskTypeController.java | 93 +++++++++++++++++++
.../api/dto/taskType/DynamicTaskInfo.java | 38 ++++++++
.../src/main/resources/application.yaml | 1 +
.../main/resources/dynamic-task-type-config.yaml | 32 +++++++
.../src/main/resources/static/shell/shell.json | 98 +++++++++++++++++++++
.../src/main/resources/static/shell/shell.png | Bin 0 -> 747 bytes
.../main/resources/static/shell/shell_hover.png | Bin 0 -> 745 bytes
.../dolphinscheduler-standalone-server.xml | 7 ++
9 files changed, 363 insertions(+)
diff --git
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/DynamicTaskTypeConfiguration.java
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/DynamicTaskTypeConfiguration.java
new file mode 100644
index 0000000000..114da5ac36
--- /dev/null
+++
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/DynamicTaskTypeConfiguration.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.configuration;
+
+import org.apache.dolphinscheduler.api.dto.taskType.DynamicTaskInfo;
+import org.apache.dolphinscheduler.common.config.YamlPropertySourceFactory;
+import org.apache.dolphinscheduler.common.constants.Constants;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import
org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.stereotype.Component;
+
+@Component
+@EnableConfigurationProperties
+@PropertySource(value = {"classpath:dynamic-task-type-config.yaml"}, factory =
YamlPropertySourceFactory.class)
+@ConfigurationProperties(prefix = "dynamic-task")
+@Getter
+@Setter
+public class DynamicTaskTypeConfiguration {
+
+ private static final Logger logger =
LoggerFactory.getLogger(DynamicTaskTypeConfiguration.class);
+ private static final List<String> defaultTaskCategories =
+ Arrays.asList(Constants.TYPE_UNIVERSAL,
Constants.TYPE_DATA_INTEGRATION, Constants.TYPE_CLOUD,
+ Constants.TYPE_LOGIC, Constants.TYPE_DATA_QUALITY,
Constants.TYPE_OTHER,
+ Constants.TYPE_MACHINE_LEARNING);
+ private List<DynamicTaskInfo> universal;
+ private List<DynamicTaskInfo> cloud;
+ private List<DynamicTaskInfo> logic;
+ private List<DynamicTaskInfo> dataIntegration;
+ private List<DynamicTaskInfo> dataQuality;
+ private List<DynamicTaskInfo> other;
+ private List<DynamicTaskInfo> machineLearning;
+
+ public List<String> getTaskCategories() {
+ return defaultTaskCategories;
+ }
+
+ public List<DynamicTaskInfo> getTaskTypesByCategory(String category) {
+ switch (category) {
+ case Constants.TYPE_UNIVERSAL:
+ return universal;
+ case Constants.TYPE_DATA_INTEGRATION:
+ return cloud;
+ case Constants.TYPE_CLOUD:
+ return logic;
+ case Constants.TYPE_LOGIC:
+ return dataIntegration;
+ case Constants.TYPE_DATA_QUALITY:
+ return dataQuality;
+ case Constants.TYPE_OTHER:
+ return other;
+ case Constants.TYPE_MACHINE_LEARNING:
+ return machineLearning;
+ default:
+ return new ArrayList<>();
+ }
+
+ }
+
+ public void printDefaultTypes() {
+ logger.info("support default universal dynamic task types: {}",
universal);
+ logger.info("support default cloud dynamic task types: {}", cloud);
+ logger.info("support default logic dynamic task types: {}", logic);
+ logger.info("support default dataIntegration dynamic task types: {}",
dataIntegration);
+ logger.info("support default dataQuality dynamic task types: {}",
dataQuality);
+ logger.info("support default machineLearning dynamic task types: {}",
machineLearning);
+ logger.info("support default other dynamic task types: {}", other);
+ }
+}
diff --git
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DynamicTaskTypeController.java
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DynamicTaskTypeController.java
new file mode 100644
index 0000000000..4dd4af6dde
--- /dev/null
+++
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DynamicTaskTypeController.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.dolphinscheduler.api.controller;
+
+import static
org.apache.dolphinscheduler.api.enums.Status.LIST_TASK_TYPE_ERROR;
+
+import org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation;
+import
org.apache.dolphinscheduler.api.configuration.DynamicTaskTypeConfiguration;
+import org.apache.dolphinscheduler.api.dto.taskType.DynamicTaskInfo;
+import org.apache.dolphinscheduler.api.enums.Status;
+import org.apache.dolphinscheduler.api.exceptions.ApiException;
+import org.apache.dolphinscheduler.api.utils.Result;
+import org.apache.dolphinscheduler.common.constants.Constants;
+import org.apache.dolphinscheduler.dao.entity.User;
+
+import java.util.List;
+
+import javax.annotation.Resource;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestAttribute;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.ResponseStatus;
+import org.springframework.web.bind.annotation.RestController;
+
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.tags.Tag;
+
+/**
+ * dynamic task type controller
+ */
+@Tag(name = "DYNAMIC_TASK_TYPE")
+@RestController
+@RequestMapping("/dynamic")
+public class DynamicTaskTypeController extends BaseController {
+
+ @Resource
+ private DynamicTaskTypeConfiguration dynamicTaskTypeConfiguration;
+
+ /**
+ * get dynamic task category list
+ *
+ * @param loginUser login user
+ * @return dynamic task category list
+ */
+ @Operation(summary = "listTaskCates", description = "LIST_TASK_TYPE_CATES")
+ @GetMapping(value = "/taskCategories")
+ @ResponseStatus(HttpStatus.OK)
+ @ApiException(LIST_TASK_TYPE_ERROR)
+ @AccessLogAnnotation(ignoreRequestArgs = "loginUser")
+ public Result listDynamicTaskCategories(@Parameter(hidden = true)
@RequestAttribute(value = Constants.SESSION_USER) User loginUser) {
+ List<String> taskCategories =
dynamicTaskTypeConfiguration.getTaskCategories();
+ return success(Status.SUCCESS.getMsg(), taskCategories);
+ }
+
+ /**
+ * get dynamic task category list
+ *
+ * @param loginUser login user
+ * @return dynamic task category list
+ */
+ @Operation(summary = "listDynamicTaskTypes", description =
"LIST_DYNAMIC_TASK_TYPES")
+ @GetMapping(value = "/{taskCategory}/taskTypes")
+ @ResponseStatus(HttpStatus.OK)
+ @ApiException(LIST_TASK_TYPE_ERROR)
+ @AccessLogAnnotation(ignoreRequestArgs = "loginUser")
+ public Result listDynamicTaskTypes(@Parameter(hidden = true)
@RequestAttribute(value = Constants.SESSION_USER) User loginUser,
+ @PathVariable("taskCategory") String
taskCategory) {
+ List<DynamicTaskInfo> taskTypes =
dynamicTaskTypeConfiguration.getTaskTypesByCategory(taskCategory);
+ return success(Status.SUCCESS.getMsg(), taskTypes);
+ }
+
+}
diff --git
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/taskType/DynamicTaskInfo.java
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/taskType/DynamicTaskInfo.java
new file mode 100644
index 0000000000..f9319e9730
--- /dev/null
+++
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/taskType/DynamicTaskInfo.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.dolphinscheduler.api.dto.taskType;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@NoArgsConstructor
+@Getter
+@Setter
+@AllArgsConstructor
+public class DynamicTaskInfo {
+
+ private String name;
+ private String hover;
+ private String icon;
+ private String json;
+
+}
diff --git a/dolphinscheduler-api/src/main/resources/application.yaml
b/dolphinscheduler-api/src/main/resources/application.yaml
index f24b158012..1a4ef1cb3f 100644
--- a/dolphinscheduler-api/src/main/resources/application.yaml
+++ b/dolphinscheduler-api/src/main/resources/application.yaml
@@ -80,6 +80,7 @@ spring:
mvc:
pathmatch:
matching-strategy: ANT_PATH_MATCHER
+ static-path-pattern: /static/**
springdoc:
swagger-ui:
path: /swagger-ui.html
diff --git
a/dolphinscheduler-api/src/main/resources/dynamic-task-type-config.yaml
b/dolphinscheduler-api/src/main/resources/dynamic-task-type-config.yaml
new file mode 100644
index 0000000000..a9a2d55ff8
--- /dev/null
+++ b/dolphinscheduler-api/src/main/resources/dynamic-task-type-config.yaml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dynamic-task:
+ universal:
+ - {name: SHELL,icon: /static/shell/shell-icon.png,hover:
/static/shell/shell-hover.png,json: /static/shell/shell.json}
+ cloud:
+ - {name: EMR,icon: shell-icon.png,hover: shell-hover.png}
+ logic:
+ - {name: SUB_PROCESS,icon: shell-icon.png,hover: shell-hover.png}
+ dataIntegration:
+ - {name: SEATUNNEL,icon: shell-icon.png,hover: shell-hover.png}
+ dataQuality:
+ - {name: DATA_QUALITY,icon: shell-icon.png,hover: shell-hover.png}
+ machineLearning:
+ - {name: JUPYTER,icon: shell-icon.png,hover: shell-hover.png}
+ other:
+ - {name: PIGEON,icon: shell-icon.png,hover: shell-hover.png}
\ No newline at end of file
diff --git a/dolphinscheduler-api/src/main/resources/static/shell/shell.json
b/dolphinscheduler-api/src/main/resources/static/shell/shell.json
new file mode 100644
index 0000000000..22d3ce7960
--- /dev/null
+++ b/dolphinscheduler-api/src/main/resources/static/shell/shell.json
@@ -0,0 +1,98 @@
+{
+ "task": "shell",
+ "locales": {
+ "zh_CN": {
+ "node_name": "节点名称",
+ "node_name_tips": "请输入节点名称",
+ "node_name_validate_message": "节点名称不能为空",
+ "script_validate_message": "脚本内容不能为空",
+ "task_priority": "任务优先级",
+ "highest": "最高",
+ "high": "高",
+ "medium": "中",
+ "low": "低",
+ "lowest": "最低",
+ "worker_group": "Worker 分组",
+ "script": "脚本"
+ },
+ "en_US": {
+ "node_name": "Node Name",
+ "node_name_tips": "Please entry node name",
+ "node_name_validate_message": "Node name cannot be empty",
+ "script_validate_message": "Script content cannot be empty",
+ "task_priority": "Task Priority",
+ "highest": "Highest",
+ "high": "High",
+ "medium": "Medium",
+ "low": "Low",
+ "lowest": "Lowest",
+ "worker_group": "Worker Group",
+ "script": "Script"
+ }
+ },
+ "apis": {
+ "getWorkerGroupList": {
+ "url": "/worker-groups/all",
+ "method": "get"
+ }
+ },
+ "forms": [
+ {
+ "label": "task_components.node_name",
+ "type": "input",
+ "field": "name",
+ "defaultValue": "",
+ "clearable": true,
+ "placeholder": "task_components.node_name_tips",
+ "validate": {
+ "required": true,
+ "trigger": ["input", "blur"],
+ "type": "non-empty",
+ "message": "task_components.node_name_validate_message"
+ }
+ },
+ {
+ "label": "task_components.task_priority",
+ "type": "select",
+ "field": "taskPriority",
+ "options": [
+ { "label": "task_components.highest", "value": "HIGHEST" },
+ { "label": "task_components.high", "value": "HIGH" },
+ { "label": "task_components.medium", "value": "MEDIUM" },
+ { "label": "task_components.low", "value": "LOW" },
+ { "label": "task_components.lowest", "value": "LOWEST" }
+ ],
+ "optionsLocale": true,
+ "defaultValue": "MEDIUM",
+ "validate": {
+ "required": true,
+ "trigger": ["input", "blur"]
+ }
+ },
+ {
+ "label": "task_components.worker_group",
+ "type": "select",
+ "field": "workerGroup",
+ "options": [],
+ "optionsLocale": false,
+ "defaultValue": "default",
+ "api": "getWorkerGroupList",
+ "validate": {
+ "required": true,
+ "trigger": ["input", "blur"]
+ }
+ },
+ {
+ "label": "task_components.script",
+ "type": "studio",
+ "field": "taskParams.rawScript",
+ "defaultValue": "",
+ "validate": {
+ "required": true,
+ "trigger": ["input", "blur"],
+ "type": "non-empty",
+ "message": "task_components.script_validate_message"
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/dolphinscheduler-api/src/main/resources/static/shell/shell.png
b/dolphinscheduler-api/src/main/resources/static/shell/shell.png
new file mode 100644
index 0000000000..4e40b6eb20
Binary files /dev/null and
b/dolphinscheduler-api/src/main/resources/static/shell/shell.png differ
diff --git
a/dolphinscheduler-api/src/main/resources/static/shell/shell_hover.png
b/dolphinscheduler-api/src/main/resources/static/shell/shell_hover.png
new file mode 100644
index 0000000000..b615f5532e
Binary files /dev/null and
b/dolphinscheduler-api/src/main/resources/static/shell/shell_hover.png differ
diff --git
a/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml
b/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml
index 14a38289f8..db312b7298 100644
---
a/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml
+++
b/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml
@@ -93,6 +93,13 @@
</includes>
<outputDirectory>conf</outputDirectory>
</fileSet>
+ <fileSet>
+
<directory>${basedir}/../dolphinscheduler-api/src/main/resources</directory>
+ <includes>
+ <include>dynamic-task-type-config.yaml</include>
+ </includes>
+ <outputDirectory>conf</outputDirectory>
+ </fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-ui/dist</directory>
<outputDirectory>./ui</outputDirectory>
