This is an automated email from the ASF dual-hosted git repository. kerwin pushed a commit to branch 3.2.0-release in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit a1cf67e19ce78510b69fbbb2379d72a8953986b9 Author: Gallardot <[email protected]> AuthorDate: Fri Sep 15 10:31:14 2023 +0800 [Bug][Worker] Global and local parameters are not replaced in SQL components (#14926) Signed-off-by: Gallardot <[email protected]> --- .../plugin/task/api/AbstractTask.java | 10 ++++- .../plugin/task/sql/SqlTaskTest.java | 45 ++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java index 10f24b7845..95ae22f272 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java @@ -38,7 +38,9 @@ public abstract class AbstractTask { protected final Logger log = LoggerFactory.getLogger(AbstractTask.class); - public String rgex = "['\"]\\$\\{(.*?)}['\"]|\\$\\{(.*?)}"; + private static String groupName1 = "paramName1"; + private static String groupName2 = "paramName2"; + public String rgex = String.format("['\"]\\$\\{(?<%s>.*?)}['\"]|\\$\\{(?<%s>.*?)}", groupName1, groupName2); /** * varPool string @@ -198,7 +200,11 @@ public abstract class AbstractTask { int index = 1; while (m.find()) { - String paramName = m.group(1); + String paramName = m.group(groupName1); + if (paramName == null) { + paramName = m.group(groupName2); + } + Property prop = paramsPropsMap.get(paramName); if (prop == null) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java index a13ef4ecc1..a46670f227 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java @@ -18,11 +18,17 @@ package org.apache.dolphinscheduler.plugin.task.sql; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; +import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.dolphinscheduler.spi.enums.DbType; +import java.util.HashMap; +import java.util.Map; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -66,6 +72,15 @@ class SqlTaskTest { String hiveLoadSql = "load inpath '/tmp/test_table/dt=${dt}' into table test_table partition(dt=${dt})"; String expected = "load inpath '/tmp/test_table/dt=?' into table test_table partition(dt=?)"; Assertions.assertEquals(expected, hiveLoadSql.replaceAll(sqlTask.rgex, "?")); + + Map<Integer, Property> sqlParamsMap = new HashMap<>(); + Map<Integer, Property> expectedSQLParamsMap = new HashMap<>(); + expectedSQLParamsMap.put(1, new Property("dt", Direct.IN, DataType.VARCHAR, "1970")); + expectedSQLParamsMap.put(2, new Property("dt", Direct.IN, DataType.VARCHAR, "1970")); + Map<String, Property> paramsMap = new HashMap<>(); + paramsMap.put("dt", new Property("dt", Direct.IN, DataType.VARCHAR, "1970")); + sqlTask.setSqlParamsMap(hiveLoadSql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); + Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); } @Test @@ -74,17 +89,37 @@ class SqlTaskTest { String expected = "select id from student where dt=?"; Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); + Map<Integer, Property> sqlParamsMap = new HashMap<>(); + Map<Integer, Property> expectedSQLParamsMap = new HashMap<>(); + expectedSQLParamsMap.put(1, new Property("dt", Direct.IN, DataType.VARCHAR, "1970")); + Map<String, Property> paramsMap = new HashMap<>(); + paramsMap.put("dt", new Property("dt", Direct.IN, DataType.VARCHAR, "1970")); + sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); + Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); + querySql = "select id from student where dt=\"${dt}\""; expected = "select id from student where dt=?"; Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); + sqlParamsMap.clear(); + sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); + Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); + querySql = "select id from student where dt=${dt}"; expected = "select id from student where dt=?"; Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); + sqlParamsMap.clear(); + sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); + Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); + querySql = "select id from student where dt=${dt} and gender=1"; expected = "select id from student where dt=? and gender=1"; Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); + + sqlParamsMap.clear(); + sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); + Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); } @Test @@ -92,5 +127,15 @@ class SqlTaskTest { String querySql = "select id from student where year=${year} and month=${month} and gender=1"; String expected = "select id from student where year=? and month=? and gender=1"; Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); + + Map<Integer, Property> sqlParamsMap = new HashMap<>(); + Map<Integer, Property> expectedSQLParamsMap = new HashMap<>(); + expectedSQLParamsMap.put(1, new Property("year", Direct.IN, DataType.VARCHAR, "1970")); + expectedSQLParamsMap.put(2, new Property("month", Direct.IN, DataType.VARCHAR, "12")); + Map<String, Property> paramsMap = new HashMap<>(); + paramsMap.put("year", new Property("year", Direct.IN, DataType.VARCHAR, "1970")); + paramsMap.put("month", new Property("month", Direct.IN, DataType.VARCHAR, "12")); + sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); + Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); } }
