This is an automated email from the ASF dual-hosted git repository.
zhoujieguang pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 16b193454b [Improvement][Task Plugin] Support Task Plugin output
response as a parameter (#13529)
16b193454b is described below
commit 16b193454b7519df712c6012eec776c90375198f
Author: JieguangZhou <[email protected]>
AuthorDate: Wed Feb 8 23:03:21 2023 +0800
[Improvement][Task Plugin] Support Task Plugin output response as a
parameter (#13529)
* support http output response as param
* add UT
* fix sqoop UT
---
docs/docs/en/guide/task/http.md | 24 ++++++++++++++++------
docs/docs/zh/guide/task/http.md | 10 +++++++++
.../task/api/parameters/AbstractParameters.java | 2 +-
.../plugin/task/http/HttpTask.java | 13 ++++++++++++
.../plugin/task/http/HttpTaskTest.java | 19 +++++++++++++++++
.../plugin/task/sqoop/EntityTestUtils.java | 5 +++++
6 files changed, 66 insertions(+), 7 deletions(-)
diff --git a/docs/docs/en/guide/task/http.md b/docs/docs/en/guide/task/http.md
index 484dde4263..d0af16eaf6 100644
--- a/docs/docs/en/guide/task/http.md
+++ b/docs/docs/en/guide/task/http.md
@@ -16,12 +16,24 @@ This node is used to perform http type tasks such as the
common POST and GET req
- Please refer to [DolphinScheduler Task Parameters Appendix](appendix.md)
`Default Task Parameters` section for default parameters.
-| **Parameter** |
**Description**
|
-|----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|---|--------------------|---|---|-------------------------|-----------------------------------------------------------------------------------------------|
-| Request address | HTTP request URL.
|
-| Request type | Supports GET, POSt, HEAD, PUT, DELETE.
|| Request parameters | Supports Parameter, Body,
Headers. || Verification conditions | Supports default response code, custom
response code, content included, content not included. |
-| Verification content | When the verification condition selects a custom
response code, the content contains, and the content does not contain, the
verification content is required. |
-| Custom parameter | It is a user-defined parameter of http part, which
will replace the content with `${variable}` in the script.
|
+| **Parameter** |
**Description**
|
+|-------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Request address | HTTP request URL.
|
+| Request type | Supports GET, POST, HEAD, PUT, DELETE.
|
+| Request parameters | Supports Parameter, Body, Headers.
|
+| Verification conditions | Supports default response code, custom response
code, content included, content not included.
|
+| Verification content | When the verification condition selects a custom
response code, the content contains, and the content does not contain, the
verification content is required. |
+| Custom parameter | It is a user-defined parameter of http part, which
will replace the content with `${variable}` in the script.
|
+
+## Task Output Parameters
+
+| **Task Parameter** | **Description** |
+|--------------------|-------------------------------------|
+| response | VARCHAR, http request return result |
+
+Can use `${taskName.response}` to reference task output parameters in
downstream tasks.
+
+For example, if the current task1 is a http task, the downstream task can use
`${task1.response}` to reference the output parameters of task1.
## Example
diff --git a/docs/docs/zh/guide/task/http.md b/docs/docs/zh/guide/task/http.md
index 5b8a774cc5..78b1231df0 100644
--- a/docs/docs/zh/guide/task/http.md
+++ b/docs/docs/zh/guide/task/http.md
@@ -26,6 +26,16 @@
| 校验内容 | 当校验条件选择自定义响应码、内容包含、内容不包含时，需填写校验内容 |
| 自定义参数 | 是 http 局部的用户自定义参数，会替换脚本中以 ${变量} 的内容 |
+## 任务输出参数
+
+| **任务参数** | **描述** |
+|----------|---------------------|
+| response | VARCHAR, http请求返回结果 |
+
+可以在下游任务中使用 ${taskName.response} 引用任务输出参数。
+
+如，当前task1为http任务, 下游任务可以使用 `${task1.response}` 引用task1的输出参数
+
## 任务样例
HTTP 定义了与服务器交互的不同方法，最基本的方法有4种，分别是GET，POST，PUT，DELETE。这里我们使用 http 任务节点，演示使用
POST 向系统的登录页面发送请求，提交数据。
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java
index 4e8e2139e6..78812fcb24 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java
@@ -199,7 +199,7 @@ public abstract class AbstractParameters implements
IParameters {
return new ResourceParametersHelper();
}
- private void addPropertyToValPool(Property property) {
+ public void addPropertyToValPool(Property property) {
varPool.removeIf(p -> p.getProp().equals(property.getProp()));
varPool.add(property);
}
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java
index a866e09876..da8b4494e3 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java
@@ -25,6 +25,8 @@ import
org.apache.dolphinscheduler.plugin.task.api.AbstractTask;
import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack;
import org.apache.dolphinscheduler.plugin.task.api.TaskException;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
+import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import
org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils;
@@ -101,6 +103,7 @@ public class HttpTask extends AbstractTask {
statusCode = String.valueOf(getStatusCode(response));
body = getResponseBody(response);
exitStatusCode = validResponse(body, statusCode);
+ addDefaultOutput(body);
long costTime = System.currentTimeMillis() - startTime;
log.info(
"startTime: {}, httpUrl: {}, httpMethod: {}, costTime : {}
milliseconds, statusCode : {}, body : {}, log : {}",
@@ -327,4 +330,14 @@ public class HttpTask extends AbstractTask {
public AbstractParameters getParameters() {
return this.httpParameters;
}
+
+ public void addDefaultOutput(String response) {
+ // put response in output
+ Property outputProperty = new Property();
+ outputProperty.setProp(String.format("%s.%s",
taskExecutionContext.getTaskName(), "response"));
+ outputProperty.setDirect(Direct.OUT);
+ outputProperty.setType(DataType.VARCHAR);
+ outputProperty.setValue(response);
+ httpParameters.addPropertyToValPool(outputProperty);
+ }
}
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java
index 5873feb019..ec908364ad 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java
@@ -21,7 +21,10 @@ import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_COD
import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_SUCCESS;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
+import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
+import
org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpStatus;
@@ -174,6 +177,22 @@ public class HttpTaskTest {
Assertions.assertEquals(EXIT_CODE_SUCCESS,
httpTask.getExitStatusCode());
}
+ @Test
+ public void testAddDefaultOutput() throws Exception {
+ HttpTask httpTask = generateHttpTask(HttpMethod.GET, HttpStatus.SC_OK);
+ AbstractParameters httpParameters = httpTask.getParameters();
+ String response = "{\"status\": \"success\"}";
+ httpTask.addDefaultOutput(response);
+
+ List<Property> varPool = httpParameters.getVarPool();
+ Assertions.assertEquals(1, varPool.size());
+ Property property = varPool.get(0);
+ Assertions.assertEquals("null.response", property.getProp());
+ Assertions.assertEquals(Direct.OUT, property.getDirect());
+ Assertions.assertEquals(DataType.VARCHAR, property.getType());
+ Assertions.assertEquals(response, property.getValue());
+ }
+
private String withMockWebServer(String path, int actualResponseCode,
String actualResponseBody) throws
IOException {
MockWebServer server = new MockWebServer();
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/EntityTestUtils.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/EntityTestUtils.java
index e1d10d1b7d..ff8c73062f 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/EntityTestUtils.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/EntityTestUtils.java
@@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.plugin.task.sqoop;
+import org.apache.commons.lang3.ObjectUtils;
+
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@@ -75,6 +77,9 @@ public class EntityTestUtils {
for (int i = 0; i < parameterTypes.length; i++) {
objects[i] =
OBJECT_MAP.get(parameterTypes[i].getName());
}
+ if (ObjectUtils.allNull(objects)) {
+ break;
+ }
method.invoke(tempInstance, objects);
} else {
method.invoke(tempInstance);
