This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new f6a6794289 [Bug][Worker] Global and local parameters are not replaced
in SQL components (#14926)
f6a6794289 is described below
commit f6a6794289ff984227548ff74affcf83fbced289
Author: Gallardot <[email protected]>
AuthorDate: Fri Sep 15 10:31:14 2023 +0800
[Bug][Worker] Global and local parameters are not replaced in SQL
components (#14926)
Signed-off-by: Gallardot <[email protected]>
---
.../plugin/task/api/AbstractTask.java | 10 ++++-
.../plugin/task/sql/SqlTaskTest.java | 45 ++++++++++++++++++++++
2 files changed, 53 insertions(+), 2 deletions(-)
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java
index 10f24b7845..95ae22f272 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java
@@ -38,7 +38,9 @@ public abstract class AbstractTask {
protected final Logger log = LoggerFactory.getLogger(AbstractTask.class);
- public String rgex = "['\"]\\$\\{(.*?)}['\"]|\\$\\{(.*?)}";
+ private static String groupName1 = "paramName1";
+ private static String groupName2 = "paramName2";
+ public String rgex =
String.format("['\"]\\$\\{(?<%s>.*?)}['\"]|\\$\\{(?<%s>.*?)}", groupName1,
groupName2);
/**
* varPool string
@@ -198,7 +200,11 @@ public abstract class AbstractTask {
int index = 1;
while (m.find()) {
- String paramName = m.group(1);
+ String paramName = m.group(groupName1);
+ if (paramName == null) {
+ paramName = m.group(groupName2);
+ }
+
Property prop = paramsPropsMap.get(paramName);
if (prop == null) {
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java
index a13ef4ecc1..a46670f227 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java
@@ -18,11 +18,17 @@
package org.apache.dolphinscheduler.plugin.task.sql;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
+import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType;
+import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import
org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters;
import
org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
import org.apache.dolphinscheduler.spi.enums.DbType;
+import java.util.HashMap;
+import java.util.Map;
+
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -66,6 +72,15 @@ class SqlTaskTest {
String hiveLoadSql = "load inpath '/tmp/test_table/dt=${dt}' into
table test_table partition(dt=${dt})";
String expected = "load inpath '/tmp/test_table/dt=?' into table
test_table partition(dt=?)";
Assertions.assertEquals(expected, hiveLoadSql.replaceAll(sqlTask.rgex,
"?"));
+
+ Map<Integer, Property> sqlParamsMap = new HashMap<>();
+ Map<Integer, Property> expectedSQLParamsMap = new HashMap<>();
+ expectedSQLParamsMap.put(1, new Property("dt", Direct.IN,
DataType.VARCHAR, "1970"));
+ expectedSQLParamsMap.put(2, new Property("dt", Direct.IN,
DataType.VARCHAR, "1970"));
+ Map<String, Property> paramsMap = new HashMap<>();
+ paramsMap.put("dt", new Property("dt", Direct.IN, DataType.VARCHAR,
"1970"));
+ sqlTask.setSqlParamsMap(hiveLoadSql, sqlTask.rgex, sqlParamsMap,
paramsMap, 1);
+ Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
}
@Test
@@ -74,17 +89,37 @@ class SqlTaskTest {
String expected = "select id from student where dt=?";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex,
"?"));
+ Map<Integer, Property> sqlParamsMap = new HashMap<>();
+ Map<Integer, Property> expectedSQLParamsMap = new HashMap<>();
+ expectedSQLParamsMap.put(1, new Property("dt", Direct.IN,
DataType.VARCHAR, "1970"));
+ Map<String, Property> paramsMap = new HashMap<>();
+ paramsMap.put("dt", new Property("dt", Direct.IN, DataType.VARCHAR,
"1970"));
+ sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap,
paramsMap, 1);
+ Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
+
querySql = "select id from student where dt=\"${dt}\"";
expected = "select id from student where dt=?";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex,
"?"));
+ sqlParamsMap.clear();
+ sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap,
paramsMap, 1);
+ Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
+
querySql = "select id from student where dt=${dt}";
expected = "select id from student where dt=?";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex,
"?"));
+ sqlParamsMap.clear();
+ sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap,
paramsMap, 1);
+ Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
+
querySql = "select id from student where dt=${dt} and gender=1";
expected = "select id from student where dt=? and gender=1";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex,
"?"));
+
+ sqlParamsMap.clear();
+ sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap,
paramsMap, 1);
+ Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
}
@Test
@@ -92,5 +127,15 @@ class SqlTaskTest {
String querySql = "select id from student where year=${year} and
month=${month} and gender=1";
String expected = "select id from student where year=? and month=? and
gender=1";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex,
"?"));
+
+ Map<Integer, Property> sqlParamsMap = new HashMap<>();
+ Map<Integer, Property> expectedSQLParamsMap = new HashMap<>();
+ expectedSQLParamsMap.put(1, new Property("year", Direct.IN,
DataType.VARCHAR, "1970"));
+ expectedSQLParamsMap.put(2, new Property("month", Direct.IN,
DataType.VARCHAR, "12"));
+ Map<String, Property> paramsMap = new HashMap<>();
+ paramsMap.put("year", new Property("year", Direct.IN,
DataType.VARCHAR, "1970"));
+ paramsMap.put("month", new Property("month", Direct.IN,
DataType.VARCHAR, "12"));
+ sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap,
paramsMap, 1);
+ Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
}
}
