This is an automated email from the ASF dual-hosted git repository.
leonbao pushed a commit to branch params-trans
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/params-trans by this push:
new 559efac [Feature-#3805][server-worker]global params of worker (#4679)
559efac is described below
commit 559efac58d7dffea5de1111e6be4c9ea00c301a3
Author: fanwq <[email protected]>
AuthorDate: Sun Feb 7 17:37:22 2021 +0800
[Feature-#3805][server-worker]global params of worker (#4679)
* return shell output to master node
* return sql output to master node
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
* fix bug
---
.../src/main/resources/datasource.properties | 2 +-
.../server/worker/runner/TaskExecuteThread.java | 1 +
.../worker/task/AbstractCommandExecutor.java | 15 +++++
.../server/worker/task/AbstractTask.java | 13 ++++
.../server/worker/task/shell/ShellTask.java | 21 +++++-
.../server/worker/task/sql/SqlTask.java | 31 +++++++--
.../worker/processor/TaskCallbackServiceTest.java | 5 ++
.../server/worker/task/TaskManagerTest.java | 75 ++++++++++++++++++++++
...ShellTaskTest.java => ShellTaskReturnTest.java} | 53 +++++++--------
.../server/worker/task/shell/ShellTaskTest.java | 18 +++++-
.../server/worker/task/sql/SqlTaskTest.java | 12 +++-
dolphinscheduler-ui/.env | 2 +-
12 files changed, 207 insertions(+), 41 deletions(-)
diff --git a/dolphinscheduler-dao/src/main/resources/datasource.properties
b/dolphinscheduler-dao/src/main/resources/datasource.properties
index 535b749..0deb7fe 100644
--- a/dolphinscheduler-dao/src/main/resources/datasource.properties
+++ b/dolphinscheduler-dao/src/main/resources/datasource.properties
@@ -66,4 +66,4 @@ spring.datasource.password=test
# open PSCache, specify count PSCache for every connection
#spring.datasource.poolPreparedStatements=true
-#spring.datasource.maxPoolPreparedStatementPerConnectionSize=20
+#spring.datasource.maxPoolPreparedStatementPerConnectionSize=20
\ No newline at end of file
diff --git
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
index 7216567..13c96c6 100644
---
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
@@ -174,6 +174,7 @@ public class TaskExecuteThread implements Runnable {
responseCommand.setProcessId(task.getProcessId());
responseCommand.setAppIds(task.getAppIds());
responseCommand.setVarPool(task.getVarPool());
+ responseCommand.setResult(task.getResultString());
logger.info("task instance id : {},task final status : {}",
taskExecutionContext.getTaskInstanceId(), task.getExitStatus());
} catch (Exception e) {
logger.error("task scheduler failure", e);
diff --git
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java
index 443bd31..52f7363 100644
---
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java
@@ -54,6 +54,7 @@ import java.util.regex.Pattern;
import org.slf4j.Logger;
+
/**
* abstract command executor
*/
@@ -85,6 +86,11 @@ public abstract class AbstractCommandExecutor {
protected final List<String> logBuffer;
/**
+ * SHELL result string
+ */
+ protected String resultString;
+
+ /**
* taskExecutionContext
*/
protected TaskExecutionContext taskExecutionContext;
@@ -223,6 +229,14 @@ public abstract class AbstractCommandExecutor {
return varPool.toString();
}
+ public String getResultString() {
+ return resultString;
+ }
+
+ public void setResultString(String result) {
+ this.resultString = result;
+ }
+
/**
* cancel application
*
@@ -355,6 +369,7 @@ public abstract class AbstractCommandExecutor {
varPool.append("$VarPool$");
} else {
logBuffer.add(line);
+ resultString = line;
lastFlushTime = flush(lastFlushTime);
}
}
diff --git
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java
index de7d35f..68152e2 100644
---
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java
@@ -64,6 +64,11 @@ public abstract class AbstractTask {
protected int processId;
/**
+ * SHELL result string
+ */
+ protected String resultString;
+
+ /**
* other resource manager appId , for example : YARN etc
*/
protected String appIds;
@@ -167,6 +172,14 @@ public abstract class AbstractTask {
this.processId = processId;
}
+ public String getResultString() {
+ return resultString;
+ }
+
+ public void setResultString(String resultString) {
+ this.resultString = resultString;
+ }
+
/**
* get task parameters
*
diff --git
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java
index 5cbd3c1..5e61f89 100644
---
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java
@@ -21,6 +21,7 @@ import static java.util.Calendar.DAY_OF_MONTH;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
+import org.apache.dolphinscheduler.common.enums.Direct;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.shell.ShellParameters;
@@ -34,6 +35,8 @@ import
org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.CommandExecuteResult;
import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor;
+import org.slf4j.Logger;
+
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -41,13 +44,13 @@ import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
+import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.slf4j.Logger;
-
/**
* shell task
*/
@@ -102,6 +105,7 @@ public class ShellTask extends AbstractTask {
setExitStatusCode(commandExecuteResult.getExitStatusCode());
setAppIds(commandExecuteResult.getAppIds());
setProcessId(commandExecuteResult.getProcessId());
+ setResult(shellCommandExecutor.getResultString());
} catch (Exception e) {
logger.error("shell task error", e);
setExitStatusCode(Constants.EXIT_CODE_FAILURE);
@@ -183,4 +187,17 @@ public class ShellTask extends AbstractTask {
}
return ParameterUtils.convertParameterPlaceholders(script,
ParamUtils.convert(paramsMap));
}
+
+ public void setResult(String result) {
+ Map<String, Property> localParams =
shellParameters.getLocalParametersMap();
+ List<Map<String, String>> outProperties = new ArrayList<>();
+ Map<String, String> p = new HashMap<>();
+ localParams.forEach((k,v) -> {
+ if (v.getDirect() == Direct.OUT) {
+ p.put(k, result);
+ }
+ });
+ outProperties.add(p);
+ resultString = JSONUtils.toJsonString(outProperties);
+ }
}
diff --git
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
index 4c328ed..cb56c0b 100644
---
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
@@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.worker.task.sql;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.DbType;
+import org.apache.dolphinscheduler.common.enums.Direct;
import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
@@ -148,7 +149,7 @@ public class SqlTask extends AbstractTask {
logger);
// execute sql task
- executeFuncAndSql(mainSqlBinds, preStatementSqlBinds,
postStatementSqlBinds, createFuncs);
+ executeFuncAndSql(mainSqlBinds, preStatementSqlBinds,
postStatementSqlBinds, createFuncs, sqlParameters.getLocalParams());
setExitStatusCode(Constants.EXIT_CODE_SUCCESS);
@@ -237,7 +238,8 @@ public class SqlTask extends AbstractTask {
public void executeFuncAndSql(SqlBinds mainSqlBinds,
List<SqlBinds> preStatementsBinds,
List<SqlBinds> postStatementsBinds,
- List<String> createFuncs) {
+ List<String> createFuncs,
+ List<Property> properties) {
Connection connection = null;
PreparedStatement stmt = null;
ResultSet resultSet = null;
@@ -253,18 +255,21 @@ public class SqlTask extends AbstractTask {
preSql(connection, preStatementsBinds);
stmt = prepareStatementAndBind(connection, mainSqlBinds);
+ String result = null;
// decide whether to executeQuery or executeUpdate based on sqlType
if (sqlParameters.getSqlType() == SqlType.QUERY.ordinal()) {
// query statements need to be convert to JsonArray and
inserted into Alert to send
resultSet = stmt.executeQuery();
- resultProcess(resultSet);
+ result = resultProcess(resultSet);
} else if (sqlParameters.getSqlType() ==
SqlType.NON_QUERY.ordinal()) {
// non query statement
- stmt.executeUpdate();
+ String updateResult = String.valueOf(stmt.executeUpdate());
+ result = setNonQuerySqlReturn(updateResult, properties);
}
postSql(connection, postStatementsBinds);
+ this.setResultString(result);
} catch (Exception e) {
logger.error("execute sql error", e);
@@ -274,13 +279,28 @@ public class SqlTask extends AbstractTask {
}
}
+ public String setNonQuerySqlReturn(String updateResult, List<Property>
properties) {
+ String result = null;
+ for (Property info :properties) {
+ if (Direct.OUT == info.getDirect()) {
+ List<Map<String,String>> updateRL = new ArrayList<>();
+ Map<String,String> updateRM = new HashMap<>();
+ updateRM.put(info.getProp(),updateResult);
+ updateRL.add(updateRM);
+ result = JSONUtils.toJsonString(updateRL);
+ break;
+ }
+ }
+ return result;
+ }
+
/**
* result process
*
* @param resultSet resultSet
* @throws Exception Exception
*/
- private void resultProcess(ResultSet resultSet) throws Exception {
+ private String resultProcess(ResultSet resultSet) throws Exception {
ArrayNode resultJSONArray = JSONUtils.createArrayNode();
ResultSetMetaData md = resultSet.getMetaData();
int num = md.getColumnCount();
@@ -300,6 +320,7 @@ public class SqlTask extends AbstractTask {
sendAttachment(sqlParameters.getGroupId(),
StringUtils.isNotEmpty(sqlParameters.getTitle()) ? sqlParameters.getTitle() :
taskExecutionContext.getTaskName() + " query result sets",
JSONUtils.toJsonString(resultJSONArray));
+ return result;
}
/**
diff --git
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java
index 5a5561d..c4e9e9f 100644
---
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java
+++
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java
@@ -119,6 +119,11 @@ public class TaskCallbackServiceTest {
ackCommand.setStartTime(new Date());
taskCallbackService.sendAck(1, ackCommand.convert2Command());
+ TaskExecuteResponseCommand responseCommand = new
TaskExecuteResponseCommand();
+ String result = responseCommand.getResult();
+ responseCommand.setResult("return string");
+ taskCallbackService.sendResult(1, responseCommand.convert2Command());
+
Stopper.stop();
nettyRemotingServer.close();
diff --git
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/TaskManagerTest.java
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/TaskManagerTest.java
index 6acfd18..24ed5b9 100644
---
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/TaskManagerTest.java
+++
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/TaskManagerTest.java
@@ -17,13 +17,22 @@
package org.apache.dolphinscheduler.server.worker.task;
+import org.apache.dolphinscheduler.common.process.Property;
+import org.apache.dolphinscheduler.common.task.sql.SqlParameters;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+import org.apache.dolphinscheduler.server.entity.SQLTaskExecutionContext;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import
org.apache.dolphinscheduler.server.worker.cache.impl.TaskExecutionContextCacheManagerImpl;
+import org.apache.dolphinscheduler.server.worker.task.shell.ShellTask;
+import org.apache.dolphinscheduler.server.worker.task.sql.SqlTask;
import org.apache.dolphinscheduler.service.alert.AlertClientService;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.junit.Assert;
import org.junit.Before;
@@ -116,4 +125,70 @@ public class TaskManagerTest {
taskExecutionContext.setTaskType("XXX");
TaskManager.newTask(taskExecutionContext,taskLogger,alertClientService);
}
+
+ @Test
+ public void testShellTaskReturnString() {
+ taskExecutionContext.setTaskInstanceId(1);
+ taskExecutionContext.setTaskName("kris test");
+ taskExecutionContext.setTaskType("SHELL");
+ taskExecutionContext.setHost("127.0.0.1:1234");
+ taskExecutionContext.setExecutePath("/tmp");
+ taskExecutionContext.setLogPath("/log");
+ taskExecutionContext.setTaskJson(
+
"{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\","
+ +
"\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\""
+ + "tasks-16849\",\"maxRetryTimes\":0,\"name\":\"shell
test 001\","
+ +
"\"params\":\"{\\\"rawScript\\\":\\\"#!/bin/sh\\\\necho $[yyyy-MM-dd HH:mm:ss
+3]\\\\necho \\\\\\\" ?? "
+ + "${time1} \\\\\\\"\\\\necho \\\\\\\" ?????
${time2}\\\\\\\"\\\\n\\\","
+ +
"\\\"localParams\\\":[{\\\"prop\\\":\\\"time1\\\",\\\"direct\\\":\\\"OUT\\\",\\\"type\\\":"
+ + "\\\"VARCHAR\\\",\\\"value\\\":\\\"$[yyyy-MM-dd
HH:mm:ss]\\\"},"
+ +
"{\\\"prop\\\":\\\"time2\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"${time_gb}\\\"}"
+ +
"],\\\"resourceList\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":"
+ +
"{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SHELL\",\"workerGroup\":\"default\"}");
+ taskExecutionContext.setProcessInstanceId(1);
+
taskExecutionContext.setGlobalParams("[{\"direct\":\"IN\",\"prop\":\"time_gb\",\"type\":\"VARCHAR\",\"value\":\"2020-12-16
17:18:33\"}]");
+ taskExecutionContext.setExecutorId(1);
+ taskExecutionContext.setCmdTypeIfComplement(5);
+ taskExecutionContext.setTenantCode("roo");
+ taskExecutionContext.setScheduleTime(new Date());
+ taskExecutionContext.setQueue("default");
+ taskExecutionContext.setTaskParams(
+ "{\"rawScript\":\"#!/bin/sh\\necho $[yyyy-MM-dd HH:mm:ss
+3]\\necho \\\" ?? ${time1} \\\"\\necho \\\" ?????
${time2}\\\"\\n\",\"localParams\":"
+ +
+
"[{\"prop\":\"time1\",\"direct\":\"OUT\",\"type\":\"VARCHAR\",\"value\":\"$[yyyy-MM-dd
HH:mm:ss]\"},{\"prop\":\"time2\",\"direct\":\"IN\",\"type\":\"VARCHAR"
+ +
"\",\"value\":\"${time_gb}\"}],\"resourceList\":[]}");
+ Map<String, String> definedParams = new HashMap<>();
+ definedParams.put("time_gb", "2020-12-16 00:00:00");
+ taskExecutionContext.setDefinedParams(definedParams);
+ ShellTask shellTask = (ShellTask)
TaskManager.newTask(taskExecutionContext,taskLogger,alertClientService);
+ shellTask.setResultString("shell return");
+ String shellReturn = shellTask.getResultString();
+ shellTask.init();
+ shellTask.setResult(shellReturn);
+ Assert.assertSame(shellReturn, "shell return");
+ }
+
+ @Test
+ public void testSqlTaskReturnString() {
+ String params =
"{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:mysql://127.0.0.1:3306\","
+ +
"\"database\":\"test\",\"jdbcUrl\":\"jdbc:mysql://127.0.0.1:3306/test\"}";
+ taskExecutionContext = new TaskExecutionContext();
+
taskExecutionContext.setTaskParams("{\"localParams\":[{\"prop\":\"ret\",
\"direct\":\"OUT\", \"type\":\"VARCHAR\", \"value\":\"\"}],"
+ + "\"type\":\"POSTGRESQL\",\"datasource\":1,\"sql\":\"insert
into tb_1 values('1','2')\","
+ + "\"sqlType\":1}");
+ taskExecutionContext.setExecutePath("/tmp");
+ taskExecutionContext.setTaskAppId("1");
+ taskExecutionContext.setTenantCode("root");
+ taskExecutionContext.setStartTime(new Date());
+ taskExecutionContext.setTaskTimeout(10000);
+ taskExecutionContext.setLogPath("/tmp/dx");
+
+ SQLTaskExecutionContext sqlTaskExecutionContext = new
SQLTaskExecutionContext();
+ sqlTaskExecutionContext.setConnectionParams(params);
+
taskExecutionContext.setSqlTaskExecutionContext(sqlTaskExecutionContext);
+ SqlTask sqlTask = new SqlTask(taskExecutionContext, logger, null);
+ SqlParameters sqlParameters =
JSONUtils.parseObject(taskExecutionContext.getTaskParams(),
SqlParameters.class);
+ List<Property> properties = sqlParameters.getLocalParams();
+ sqlTask.setNonQuerySqlReturn("sql return", properties);
+ }
}
diff --git
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskReturnTest.java
similarity index 63%
copy from
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java
copy to
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskReturnTest.java
index c5f2de8..1af0a38 100644
---
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java
+++
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskReturnTest.java
@@ -39,12 +39,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * shell task test.
+ * shell task return test.
*/
@RunWith(PowerMockRunner.class)
@PrepareForTest({ShellTask.class})
-public class ShellTaskTest {
-
+public class ShellTaskReturnTest {
private static final Logger logger =
LoggerFactory.getLogger(ShellTaskTest.class);
private ShellTask shellTask;
@@ -65,13 +64,19 @@ public class ShellTaskTest {
taskExecutionContext.setExecutePath("/tmp");
taskExecutionContext.setLogPath("/log");
taskExecutionContext.setTaskJson(
-
"{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\""
- +
- "tasks-16849\",\"maxRetryTimes\":0,\"name\":\"shell test
001\",\"params\":\"{\\\"rawScript\\\":\\\"#!/bin/sh\\\\necho $[yyyy-MM-dd
HH:mm:ss +3]\\\\necho \\\\\\\" ?? "
- + "${time1} \\\\\\\"\\\\necho \\\\\\\" ?????
${time2}\\\\\\\"\\\\n\\\",\\\"localParams\\\":[{\\\"prop\\\":\\\"time1\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":"
- + "\\\"VARCHAR\\\",\\\"value\\\":\\\"$[yyyy-MM-dd
HH:mm:ss]\\\"},{\\\"prop\\\":\\\"time2\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"${time_gb}\\\"}"
- +
"],\\\"resourceList\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":"
- +
"{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SHELL\",\"workerGroup\":\"default\"}");
+ "{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],"
+ +
"\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,"
+ +
"\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\""
+ + "tasks-16849\",\"maxRetryTimes\":0,\"name\":\"shell
test 001\","
+ +
"\"params\":\"{\\\"rawScript\\\":\\\"#!/bin/sh\\\\necho $[yyyy-MM-dd HH:mm:ss
+3]\\\\necho \\\\\\\" ?? "
+ + "${time1} \\\\\\\"\\\\necho \\\\\\\" ?????
${time2}\\\\\\\"\\\\n\\\","
+ +
"\\\"localParams\\\":[{\\\"prop\\\":\\\"time1\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":"
+ + "\\\"VARCHAR\\\",\\\"value\\\":\\\"$[yyyy-MM-dd
HH:mm:ss]\\\"},"
+ +
"{\\\"prop\\\":\\\"time2\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"${time_gb}\\\"}"
+ +
"],\\\"resourceList\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,"
+ +
"\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":"
+ +
"{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\","
+ + "\"type\":\"SHELL\",\"workerGroup\":\"default\"}");
taskExecutionContext.setProcessInstanceId(1);
taskExecutionContext.setGlobalParams("[{\"direct\":\"IN\",\"prop\":\"time_gb\",\"type\":\"VARCHAR\",\"value\":\"2020-12-16
17:18:33\"}]");
taskExecutionContext.setExecutorId(1);
@@ -80,10 +85,10 @@ public class ShellTaskTest {
taskExecutionContext.setScheduleTime(new Date());
taskExecutionContext.setQueue("default");
taskExecutionContext.setTaskParams(
- "{\"rawScript\":\"#!/bin/sh\\necho $[yyyy-MM-dd HH:mm:ss
+3]\\necho \\\" ?? ${time1} \\\"\\necho \\\" ?????
${time2}\\\"\\n\",\"localParams\":"
- +
-
"[{\"prop\":\"time1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"$[yyyy-MM-dd
HH:mm:ss]\"},{\"prop\":\"time2\",\"direct\":\"IN\",\"type\":\"VARCHAR"
- + "\",\"value\":\"${time_gb}\"}],\"resourceList\":[]}");
+ "{\"rawScript\":\"#!/bin/sh\\necho $[yyyy-MM-dd HH:mm:ss
+3]\\necho \\\" ?? ${time1} \\\"\\necho \\\" ?????
${time2}\\\"\\n\",\"localParams\":"
+ +
+
"[{\"prop\":\"time1\",\"direct\":\"OUT\",\"type\":\"VARCHAR\",\"value\":\"$[yyyy-MM-dd
HH:mm:ss]\"},{\"prop\":\"time2\",\"direct\":\"IN\",\"type\":\"VARCHAR"
+ +
"\",\"value\":\"${time_gb}\"}],\"resourceList\":[]}");
Map<String, String> definedParams = new HashMap<>();
definedParams.put("time_gb", "2020-12-16 00:00:00");
taskExecutionContext.setDefinedParams(definedParams);
@@ -96,19 +101,15 @@ public class ShellTaskTest {
}
@Test
- public void testComplementData() throws Exception {
- shellTask = new ShellTask(taskExecutionContext, logger);
- shellTask.init();
-
PowerMockito.when(shellCommandExecutor.run(anyString())).thenReturn(commandExecuteResult);
- shellTask.handle();
- }
-
- @Test
- public void testStartProcess() throws Exception {
- taskExecutionContext.setCmdTypeIfComplement(0);
+ public void testShellReturnString() {
shellTask = new ShellTask(taskExecutionContext, logger);
shellTask.init();
-
PowerMockito.when(shellCommandExecutor.run(anyString())).thenReturn(commandExecuteResult);
- shellTask.handle();
+ try {
+
PowerMockito.when(shellCommandExecutor.run(anyString())).thenReturn(commandExecuteResult);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ shellTask.setResult("shell return string");
+ logger.info("shell return string:{}", shellTask.getResultString());
}
}
diff --git
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java
index c5f2de8..a8e9e70 100644
---
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java
+++
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java
@@ -19,12 +19,15 @@ package
org.apache.dolphinscheduler.server.worker.task.shell;
import static org.mockito.ArgumentMatchers.anyString;
+import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.server.worker.task.CommandExecuteResult;
import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import java.nio.file.Files;
import java.nio.file.Paths;
+import java.sql.DriverManager;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
@@ -42,7 +45,7 @@ import org.slf4j.LoggerFactory;
* shell task test.
*/
@RunWith(PowerMockRunner.class)
-@PrepareForTest({ShellTask.class})
+@PrepareForTest(value = {ShellTask.class, DriverManager.class,
SpringApplicationContext.class, ParameterUtils.class})
public class ShellTaskTest {
private static final Logger logger =
LoggerFactory.getLogger(ShellTaskTest.class);
@@ -57,6 +60,7 @@ public class ShellTaskTest {
System.setProperty("log4j2.disable.jmx", Boolean.TRUE.toString());
shellCommandExecutor = PowerMockito.mock(ShellCommandExecutor.class);
PowerMockito.whenNew(ShellCommandExecutor.class).withAnyArguments().thenReturn(shellCommandExecutor);
+ shellCommandExecutor.setResultString("shellReturn");
taskExecutionContext = new TaskExecutionContext();
taskExecutionContext.setTaskInstanceId(1);
taskExecutionContext.setTaskName("kris test");
@@ -68,7 +72,7 @@ public class ShellTaskTest {
"{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\""
+
"tasks-16849\",\"maxRetryTimes\":0,\"name\":\"shell test
001\",\"params\":\"{\\\"rawScript\\\":\\\"#!/bin/sh\\\\necho $[yyyy-MM-dd
HH:mm:ss +3]\\\\necho \\\\\\\" ?? "
- + "${time1} \\\\\\\"\\\\necho \\\\\\\" ?????
${time2}\\\\\\\"\\\\n\\\",\\\"localParams\\\":[{\\\"prop\\\":\\\"time1\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":"
+ + "${time1} \\\\\\\"\\\\necho \\\\\\\" ?????
${time2}\\\\\\\"\\\\n\\\",\\\"localParams\\\":[{\\\"prop\\\":\\\"time1\\\",\\\"direct\\\":\\\"OUT\\\",\\\"type\\\":"
+ "\\\"VARCHAR\\\",\\\"value\\\":\\\"$[yyyy-MM-dd
HH:mm:ss]\\\"},{\\\"prop\\\":\\\"time2\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"${time_gb}\\\"}"
+
"],\\\"resourceList\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":"
+
"{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SHELL\",\"workerGroup\":\"default\"}");
@@ -82,7 +86,7 @@ public class ShellTaskTest {
taskExecutionContext.setTaskParams(
"{\"rawScript\":\"#!/bin/sh\\necho $[yyyy-MM-dd HH:mm:ss
+3]\\necho \\\" ?? ${time1} \\\"\\necho \\\" ?????
${time2}\\\"\\n\",\"localParams\":"
+
-
"[{\"prop\":\"time1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"$[yyyy-MM-dd
HH:mm:ss]\"},{\"prop\":\"time2\",\"direct\":\"IN\",\"type\":\"VARCHAR"
+
"[{\"prop\":\"time1\",\"direct\":\"OUT\",\"type\":\"VARCHAR\",\"value\":\"$[yyyy-MM-dd
HH:mm:ss]\"},{\"prop\":\"time2\",\"direct\":\"IN\",\"type\":\"VARCHAR"
+ "\",\"value\":\"${time_gb}\"}],\"resourceList\":[]}");
Map<String, String> definedParams = new HashMap<>();
definedParams.put("time_gb", "2020-12-16 00:00:00");
@@ -111,4 +115,12 @@ public class ShellTaskTest {
PowerMockito.when(shellCommandExecutor.run(anyString())).thenReturn(commandExecuteResult);
shellTask.handle();
}
+
+ @Test
+ public void testSetResult() {
+ shellTask = new ShellTask(taskExecutionContext, logger);
+ shellTask.init();
+ String r = "return";
+ shellTask.setResult(r);
+ }
}
diff --git
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTaskTest.java
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTaskTest.java
index 64db568..2abb91c 100644
---
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTaskTest.java
+++
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTaskTest.java
@@ -19,10 +19,12 @@ package org.apache.dolphinscheduler.server.worker.task.sql;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
+import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.server.entity.SQLTaskExecutionContext;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
import org.apache.dolphinscheduler.service.alert.AlertClientService;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -44,7 +46,7 @@ import org.slf4j.LoggerFactory;
* sql task test
*/
@RunWith(PowerMockRunner.class)
-@PrepareForTest(value = {SqlTask.class, DriverManager.class})
+@PrepareForTest(value = {SqlTask.class, DriverManager.class,
SpringApplicationContext.class, ParameterUtils.class})
public class SqlTaskTest {
private static final Logger logger =
LoggerFactory.getLogger(SqlTaskTest.class);
@@ -70,7 +72,9 @@ public class SqlTaskTest {
props.setTaskStartTime(new Date());
props.setTaskTimeout(0);
props.setTaskParams(
-
"{\"localParams\":[],\"type\":\"POSTGRESQL\",\"datasource\":1,\"sql\":\"insert
into tb_1 values('1','2')\",\"sqlType\":1}");
+ "{\"localParams\":[{\"prop\":\"ret\", \"direct\":\"OUT\",
\"type\":\"VARCHAR\", \"value\":\"\"}],"
+ +
"\"type\":\"POSTGRESQL\",\"datasource\":1,\"sql\":\"insert into tb_1
values('1','2')\","
+ + "\"sqlType\":1}");
taskExecutionContext = PowerMockito.mock(TaskExecutionContext.class);
PowerMockito.when(taskExecutionContext.getTaskParams()).thenReturn(props.getTaskParams());
@@ -85,6 +89,8 @@ public class SqlTaskTest {
sqlTaskExecutionContext.setConnectionParams(CONNECTION_PARAMS);
PowerMockito.when(taskExecutionContext.getSqlTaskExecutionContext()).thenReturn(sqlTaskExecutionContext);
+ PowerMockito.mockStatic(SpringApplicationContext.class);
+
PowerMockito.when(SpringApplicationContext.getBean(Mockito.any())).thenReturn(new
AlertDao());
alertClientService = PowerMockito.mock(AlertClientService.class);
sqlTask = new SqlTask(taskExecutionContext, logger,
alertClientService);
sqlTask.init();
@@ -95,7 +101,7 @@ public class SqlTaskTest {
Assert.assertNotNull(sqlTask.getParameters());
}
- @Test(expected = Exception.class)
+ @Test
public void testHandle() throws Exception {
Connection connection = PowerMockito.mock(Connection.class);
PowerMockito.mockStatic(DriverManager.class);
diff --git a/dolphinscheduler-ui/.env b/dolphinscheduler-ui/.env
index e676be6..e6b00a1 100644
--- a/dolphinscheduler-ui/.env
+++ b/dolphinscheduler-ui/.env
@@ -14,7 +14,7 @@
# limitations under the License.
# back end interface address
-API_BASE = http://192.168.xx.xx:12345
+API_BASE = http://127.0.0.1:12345
# If IP access is required for local development, remove the "#"
#DEV_HOST = 192.168.xx.xx
