This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 606b56403e [Fix-13381][plugin] fix error while parsing hive load sql
(#13378)
606b56403e is described below
commit 606b56403eadc77a46972c51f23365cc227edd38
Author: Alex Ting <[email protected]>
AuthorDate: Mon May 22 11:19:30 2023 +0800
[Fix-13381][plugin] fix error while parsing hive load sql (#13378)
* [fix][task plugin][task sql] fix parsing error while using variables
sometimes parsing exception happened if task sql contains a variable followed
by \" or \' .
for example:
input sql: load inpath '/tmp/test_table/dt=${dt}'
into table test_table partition(dt=${dt})
after replace: preparing : load inpath '/tmp/test_table/dt=?
into table test_table partition(dt=?)
---
docs/docs/en/guide/upgrade/incompatible.md | 1 +
docs/docs/zh/guide/upgrade/incompatible.md | 1 +
.../plugin/task/api/AbstractTask.java | 2 +-
.../plugin/task/sql/SqlTaskTest.java | 96 ++++++++++++++++++++++
4 files changed, 99 insertions(+), 1 deletion(-)
diff --git a/docs/docs/en/guide/upgrade/incompatible.md
b/docs/docs/en/guide/upgrade/incompatible.md
index ba1af107ce..c3106631af 100644
--- a/docs/docs/en/guide/upgrade/incompatible.md
+++ b/docs/docs/en/guide/upgrade/incompatible.md
@@ -4,6 +4,7 @@ This document records the incompatible updates between each
version. You need to
## dev
+* Change regex matching sql params in SQL task plugin
([#13378](https://github.com/apache/dolphinscheduler/pull/13378))
* Remove the spark version of spark task
([#11860](https://github.com/apache/dolphinscheduler/pull/11860)).
* Change the default unix shell executor from sh to bash
([#12180](https://github.com/apache/dolphinscheduler/pull/12180)).
diff --git a/docs/docs/zh/guide/upgrade/incompatible.md
b/docs/docs/zh/guide/upgrade/incompatible.md
index 4e294f954d..119f77e2ef 100644
--- a/docs/docs/zh/guide/upgrade/incompatible.md
+++ b/docs/docs/zh/guide/upgrade/incompatible.md
@@ -4,6 +4,7 @@
## dev
+* 更新了SQL任务中用于匹配变量的正则表达式
([#13378](https://github.com/apache/dolphinscheduler/pull/13378))
* Remove the spark version of spark task
([#11860](https://github.com/apache/dolphinscheduler/pull/11860)).
* Change the default unix shell executor from sh to bash
([#12180](https://github.com/apache/dolphinscheduler/pull/12180)).
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java
index 30f5b7d30f..10f24b7845 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java
@@ -38,7 +38,7 @@ public abstract class AbstractTask {
protected final Logger log = LoggerFactory.getLogger(AbstractTask.class);
- public String rgex = "['\"]*\\$\\{(.*?)\\}['\"]*";
+ public String rgex = "['\"]\\$\\{(.*?)}['\"]|\\$\\{(.*?)}";
/**
* varPool string
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java
new file mode 100644
index 0000000000..a13ef4ecc1
--- /dev/null
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.task.sql;
+
+import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType;
+import
org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters;
+import
org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class SqlTaskTest {
+
+ private SqlTask sqlTask;
+
+ @BeforeEach
+ void setup() {
+ DataSourceParameters parameters = new DataSourceParameters();
+ parameters.setType(DbType.HIVE);
+ parameters.setResourceType(ResourceType.DATASOURCE.name());
+
+ ResourceParametersHelper resourceParametersHelper = new
ResourceParametersHelper();
+ resourceParametersHelper.put(ResourceType.DATASOURCE, 1, parameters);
+
+ TaskExecutionContext ctx = new TaskExecutionContext();
+ ctx.setResourceParametersHelper(resourceParametersHelper);
+
ctx.setTaskParams("{\"type\":\"HIVE\",\"datasource\":1,\"sql\":\"select 1\"}");
+
+ sqlTask = new SqlTask(ctx);
+ }
+
+ @Test
+ void testReplacingSqlWithoutParams() {
+ String querySql = "select 1";
+ String expected = "select 1";
+ Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex,
"?"));
+ }
+
+ @Test
+ void testReplacingSqlWithDollarSymbol() {
+ String querySql = "select concat(amount, '$') as price from product";
+ String expected = "select concat(amount, '$') as price from product";
+ Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex,
"?"));
+ }
+
+ @Test
+ void testReplacingHiveLoadSql() {
+ String hiveLoadSql = "load inpath '/tmp/test_table/dt=${dt}' into
table test_table partition(dt=${dt})";
+ String expected = "load inpath '/tmp/test_table/dt=?' into table
test_table partition(dt=?)";
+ Assertions.assertEquals(expected, hiveLoadSql.replaceAll(sqlTask.rgex,
"?"));
+ }
+
+ @Test
+ void testReplacingSelectSql() {
+ String querySql = "select id from student where dt='${dt}'";
+ String expected = "select id from student where dt=?";
+ Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex,
"?"));
+
+ querySql = "select id from student where dt=\"${dt}\"";
+ expected = "select id from student where dt=?";
+ Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex,
"?"));
+
+ querySql = "select id from student where dt=${dt}";
+ expected = "select id from student where dt=?";
+ Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex,
"?"));
+
+ querySql = "select id from student where dt=${dt} and gender=1";
+ expected = "select id from student where dt=? and gender=1";
+ Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex,
"?"));
+ }
+
+ @Test
+ void testReplacingSqlNonGreedy() {
+ String querySql = "select id from student where year=${year} and
month=${month} and gender=1";
+ String expected = "select id from student where year=? and month=? and
gender=1";
+ Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex,
"?"));
+ }
+}
