(dolphinscheduler) branch dev updated: [Fix-17023] Fix split and remove comment error in Doris task. (#17070)

[wenjun]

This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git


The following commit(s) were added to refs/heads/dev by this push:
     new 1b28cd60d7 [Fix-17023] Fix split and remove comment error in Doris 
task. (#17070)
1b28cd60d7 is described below

commit 1b28cd60d76818535254126faa96530b754867d7
Author: ZHANG YINGHONG <joy.zha...@outlook.com>
AuthorDate: Mon Mar 24 14:04:59 2025 +0800

    [Fix-17023] Fix split and remove comment error in Doris task. (#17070)
---
 .../plugin/doris/param/DorisDataSourceProcessor.java          |  9 +++++++++
 .../plugin/doris/param/DorisDataSourceProcessorTest.java      | 11 +++++++++++
 2 files changed, 20 insertions(+)

diff --git 
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessor.java
 
b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessor.java
index e4322e3bee..d948d8876b 100644
--- 
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessor.java
+++ 
b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessor.java
@@ -37,6 +37,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import com.alibaba.druid.sql.parser.SQLParserUtils;
 import com.google.auto.service.AutoService;
 
 @AutoService(DataSourceProcessor.class)
@@ -139,6 +140,14 @@ public class DorisDataSourceProcessor extends 
AbstractDataSourceProcessor {
         return new DorisDataSourceProcessor();
     }
 
+    @Override
+    public List<String> splitAndRemoveComment(String sql) {
+        // Because doris is highly compatible with mysql syntax,
+        // a mysql-type implementation can be used for comments and statement 
split.
+        String cleanSQL = SQLParserUtils.removeComment(sql, 
com.alibaba.druid.DbType.mysql);
+        return SQLParserUtils.split(cleanSQL, com.alibaba.druid.DbType.mysql);
+    }
+
     private String transformOther(Map<String, String> paramMap) {
         if (MapUtils.isEmpty(paramMap)) {
             return null;
diff --git 
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessorTest.java
 
b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessorTest.java
index 168a200e23..192429689c 100644
--- 
a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessorTest.java
+++ 
b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessorTest.java
@@ -137,4 +137,15 @@ public class DorisDataSourceProcessorTest {
         Assertions.assertEquals("set enable_unique_key_partial_update = 
false", sqls.get(2));
     }
 
+    @Test
+    void testSplitAndRemoveCommentWithComments() {
+        String sql = "-- This is a comment\r\n" +
+                "select *, udfTest(p1) from table1 -- This is a comment\r\n;" +
+                "/*Annotation test*/select * from table2/*Annotation test*/";
+        List<String> result = 
dorisDatasourceProcessor.splitAndRemoveComment(sql);
+        Assertions.assertEquals(2, result.size());
+        Assertions.assertEquals("select *, udfTest(p1) from table1", 
result.get(0));
+        Assertions.assertEquals("select * from table2", result.get(1));
+    }
+
 }