This is an automated email from the ASF dual-hosted git repository.
wenjun pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 020092ed1e Set charset in upgrading tools to solve read upgrade sql
mojibake (#14955)
020092ed1e is described below
commit 020092ed1e8328e76a27d06632337bb3e0324f5a
Author: Wenjun Ruan <[email protected]>
AuthorDate: Tue Sep 26 09:46:25 2023 +0800
Set charset in upgrading tools to solve read upgrade sql mojibake (#14955)
---
.../common/sql/ClasspathSqlScriptParser.java | 137 +++++++++++++++++
.../common/sql/SqlScriptParser.java} | 23 +--
.../common/sql/SqlScriptRunner.java | 65 +++++++++
.../common/utils/ScriptRunner.java | 162 ---------------------
.../common/sql/ClasspathSqlScriptParserTest.java | 52 +++++++
.../src/test/resources/sql/mysql_ddl.sql | 44 ++++++
.../src/test/resources/sql/mysql_dml.sql | 84 +++++++++++
.../tools/datasource/dao/UpgradeDao.java | 54 +++----
8 files changed, 408 insertions(+), 213 deletions(-)
diff --git
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/sql/ClasspathSqlScriptParser.java
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/sql/ClasspathSqlScriptParser.java
new file mode 100644
index 0000000000..3b4b3765f3
--- /dev/null
+++
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/sql/ClasspathSqlScriptParser.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.common.sql;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.LineNumberReader;
+import java.io.Reader;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+
+public class ClasspathSqlScriptParser implements SqlScriptParser {
+
+ private final String sqlScriptPath;
+
+ private final Charset charset;
+
+ public ClasspathSqlScriptParser(String sqlScriptPath) {
+ this.sqlScriptPath = sqlScriptPath;
+ this.charset = StandardCharsets.UTF_8;
+ }
+
+ @Override
+ public List<String> getAllSql() throws IOException {
+ Resource sqlScriptResource = new ClassPathResource(sqlScriptPath);
+ List<String> result = new ArrayList<>();
+ try (
+ InputStream inputStream = sqlScriptResource.getInputStream();
+ Reader sqlScriptReader = new InputStreamReader(inputStream,
charset);
+ LineNumberReader lineNumberReader = new
LineNumberReader(sqlScriptReader)) {
+ String sql;
+ do {
+ sql = parseNextSql(lineNumberReader);
+ if (StringUtils.isNotBlank(sql)) {
+ result.add(sql);
+ }
+ } while (StringUtils.isNotBlank(sql));
+ }
+ return result;
+ }
+
+ private String parseNextSql(LineNumberReader lineNumberReader) throws
IOException {
+ String line;
+ while ((line = lineNumberReader.readLine()) != null) {
+ String trimLine = line.trim();
+ if (StringUtils.isEmpty(trimLine) || isComment(trimLine)) {
+ // Skip the empty line, comment line
+ continue;
+ }
+ if (trimLine.startsWith("/*")) {
+ skipLicenseHeader(lineNumberReader);
+ continue;
+ }
+ if (trimLine.startsWith("delimiter")) {
+ // begin to parse processor, until delimiter ;
+ String[] split = trimLine.split(" ");
+ return parseProcedure(lineNumberReader, split[1]);
+ }
+ // begin to parse sql until;
+ List<String> sqlLines = new ArrayList<>();
+ sqlLines.add(line);
+ while (!line.endsWith(";")) {
+ line = lineNumberReader.readLine();
+ if (line == null) {
+ break;
+ }
+ if (StringUtils.isBlank(line)) {
+ continue;
+ }
+ sqlLines.add(line);
+ }
+ return String.join("\n", sqlLines);
+ }
+ return null;
+ }
+
+ private void skipLicenseHeader(LineNumberReader lineNumberReader) throws
IOException {
+ String line;
+ while ((line = lineNumberReader.readLine()) != null) {
+ String trimLine = line.trim();
+ if (StringUtils.isEmpty(trimLine) || isComment(trimLine)) {
+ // Skip the empty line, comment line
+ continue;
+ }
+ if (line.startsWith("*/")) {
+ break;
+ }
+ }
+ }
+
+ private String parseProcedure(LineNumberReader lineNumberReader, String
delimiter) throws IOException {
+ List<String> sqlLines = new ArrayList<>();
+ // begin to parse processor, until delimiter ;
+ String line;
+ while (true) {
+ line = lineNumberReader.readLine();
+ if (line == null) {
+ break;
+ }
+ if (StringUtils.isBlank(line)) {
+ continue;
+ }
+ if (line.startsWith(delimiter)) {
+ break;
+ }
+ sqlLines.add(line);
+ }
+ return String.join("\n", sqlLines);
+ }
+
+ private boolean isComment(String line) {
+ return line.startsWith("--") || line.startsWith("//");
+ }
+}
diff --git
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/sql/SqlScriptParser.java
similarity index 59%
rename from
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java
rename to
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/sql/SqlScriptParser.java
index 7b1b96b56f..3e034c7cc2 100644
---
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java
+++
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/sql/SqlScriptParser.java
@@ -14,25 +14,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.utils;
-import java.io.StringReader;
+package org.apache.dolphinscheduler.common.sql;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
+import java.io.IOException;
+import java.util.List;
-public class ScriptRunnerTest {
+public interface SqlScriptParser {
+
+ List<String> getAllSql() throws IOException;
- @Test
- public void testRunScript() {
- // connection is null
- Exception exception = null;
- ScriptRunner s = new ScriptRunner(null, true, true);
- try {
- s.runScript(new StringReader("select 1"));
- } catch (Exception e) {
- exception = e;
- }
- Assertions.assertNotNull(exception);
- }
}
diff --git
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/sql/SqlScriptRunner.java
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/sql/SqlScriptRunner.java
new file mode 100644
index 0000000000..ab16bf2ac0
--- /dev/null
+++
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/sql/SqlScriptRunner.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.common.sql;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.List;
+
+import javax.sql.DataSource;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class SqlScriptRunner {
+
+ private final DataSource dataSource;
+
+ private final SqlScriptParser sqlScriptParser;
+
+ /**
+ * @param dataSource DataSource which used to execute the sql
script.
+ * @param sqlScriptFilePath Sqk script file path, the path should under
classpath.
+ */
+ public SqlScriptRunner(DataSource dataSource, String sqlScriptFilePath) {
+ this.dataSource = dataSource;
+ this.sqlScriptParser = new ClasspathSqlScriptParser(sqlScriptFilePath);
+ }
+
+ public void execute() throws SQLException, IOException {
+ List<String> allSql = sqlScriptParser.getAllSql();
+ try (Connection connection = dataSource.getConnection()) {
+ for (String sql : allSql) {
+ if (StringUtils.isBlank(sql)) {
+ continue;
+ }
+ try (Statement statement = connection.createStatement()) {
+ // Since some sql doesn't have result so we believe if
there is no exception then we think the sql
+ // execute success.
+ statement.execute(sql);
+ log.info("Execute sql: {} success", sql);
+ }
+ }
+ }
+ }
+
+}
diff --git
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java
deleted file mode 100644
index 6bef7ce894..0000000000
---
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Copyright 2004-2020 the original author or authors.
- * <p>
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.common.utils;
-
-import java.io.IOException;
-import java.io.LineNumberReader;
-import java.io.Reader;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.List;
-
-import lombok.extern.slf4j.Slf4j;
-
-/**
- * Tool to run database scripts
- */
-@Slf4j
-public class ScriptRunner {
-
- private static final String DEFAULT_DELIMITER = ";";
-
- private final Connection connection;
-
- private final boolean stopOnError;
- private final boolean autoCommit;
-
- private String delimiter = DEFAULT_DELIMITER;
- private boolean fullLineDelimiter = false;
-
- public ScriptRunner(Connection connection, boolean autoCommit, boolean
stopOnError) {
- this.connection = connection;
- this.autoCommit = autoCommit;
- this.stopOnError = stopOnError;
- }
-
- public void setDelimiter(String delimiter, boolean fullLineDelimiter) {
- this.delimiter = delimiter;
- this.fullLineDelimiter = fullLineDelimiter;
- }
-
- /**
- * Runs an SQL script (read in using the Reader parameter)
- *
- * @param reader - the source of the script
- * @throws IOException errors
- * @throws SQLException errors
- */
- public void runScript(Reader reader) throws IOException, SQLException {
- try {
- boolean originalAutoCommit = connection.getAutoCommit();
- try {
- if (originalAutoCommit != this.autoCommit) {
- connection.setAutoCommit(this.autoCommit);
- }
- runScript(connection, reader);
- } finally {
- connection.setAutoCommit(originalAutoCommit);
- }
- } catch (IOException | SQLException e) {
- throw e;
- } catch (Exception e) {
- throw new RuntimeException("Error running script. Cause: " + e,
e);
- }
- }
-
- /**
- * Runs an SQL script (read in using the Reader parameter) using the
connection
- * passed in
- *
- * @param conn - the connection to use for the script
- * @param reader - the source of the script
- * @throws SQLException if any SQL errors occur
- * @throws IOException if there is an error reading from the Reader
- */
- private void runScript(Connection conn, Reader reader) throws IOException,
SQLException {
- List<String> command = null;
- try {
- LineNumberReader lineReader = new LineNumberReader(reader);
- String line;
- while ((line = lineReader.readLine()) != null) {
- if (command == null) {
- command = new ArrayList<>();
- }
- String trimmedLine = line.trim();
- if (trimmedLine.startsWith("--")) {
- log.info("\n{}", trimmedLine);
- } else if (trimmedLine.length() < 1 ||
trimmedLine.startsWith("//")) {
- // Do nothing
- } else if (trimmedLine.startsWith("delimiter")) {
- String newDelimiter = trimmedLine.split(" ")[1];
- this.setDelimiter(newDelimiter, fullLineDelimiter);
-
- } else if (!fullLineDelimiter &&
trimmedLine.endsWith(getDelimiter())
- || fullLineDelimiter &&
trimmedLine.equals(getDelimiter())) {
- command.add(line.substring(0,
line.lastIndexOf(getDelimiter())));
- log.info("\n{}", String.join("\n", command));
-
- try (Statement statement = conn.createStatement()) {
- statement.execute(String.join(" ", command));
- try (ResultSet rs = statement.getResultSet()) {
- if (stopOnError && rs != null) {
- ResultSetMetaData md = rs.getMetaData();
- int cols = md.getColumnCount();
- for (int i = 1; i < cols; i++) {
- String name = md.getColumnLabel(i);
- log.info("{} \t", name);
- }
- log.info("");
- while (rs.next()) {
- for (int i = 1; i < cols; i++) {
- String value = rs.getString(i);
- log.info("{} \t", value);
- }
- log.info("");
- }
- }
- }
- } catch (SQLException e) {
- log.error("SQLException", e);
- throw e;
- }
-
- command = null;
- Thread.yield();
- } else {
- command.add(line);
- }
- }
-
- } catch (SQLException e) {
- log.error("Error executing: {}", command);
- throw e;
- } catch (IOException e) {
- e.fillInStackTrace();
- log.error("Error executing: {}", command);
- throw e;
- }
- }
-
- private String getDelimiter() {
- return delimiter;
- }
-
-}
diff --git
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/sql/ClasspathSqlScriptParserTest.java
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/sql/ClasspathSqlScriptParserTest.java
new file mode 100644
index 0000000000..e2a23845ad
--- /dev/null
+++
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/sql/ClasspathSqlScriptParserTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.common.sql;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class ClasspathSqlScriptParserTest {
+
+ @Test
+ void testMysqlDmlSql() throws IOException {
+ ClasspathSqlScriptParser classpathSqlScriptParser = new
ClasspathSqlScriptParser("sql/mysql_dml.sql");
+ List<String> allSql = classpathSqlScriptParser.getAllSql();
+ Assertions.assertEquals("drop PROCEDURE if EXISTS
dolphin_t_ds_tenant_insert_default;", allSql.get(0));
+ Assertions.assertEquals("CREATE PROCEDURE
dolphin_t_ds_tenant_insert_default()\n" +
+ "BEGIN\n" +
+ " IF\n" +
+ "NOT EXISTS(SELECT 1\n" +
+ " FROM t_ds_tenant\n" +
+ " WHERE id = -1)\n" +
+ " THEN\n" +
+ " INSERT INTO `t_ds_tenant` VALUES ('-1', 'default',
'default tenant', '1', current_timestamp, current_timestamp);\n"
+ +
+ "END IF;\n" +
+ "END;", String.join("", allSql.get(1)));
+ }
+
+ @Test
+ void testMysqlDdlSql() throws IOException {
+ ClasspathSqlScriptParser classpathSqlScriptParser = new
ClasspathSqlScriptParser("sql/mysql_ddl.sql");
+ List<String> allSql = classpathSqlScriptParser.getAllSql();
+ Assertions.assertEquals("ALTER TABLE t_ds_process_definition DROP
tenant_id;", allSql.get(0));
+ }
+}
diff --git a/dolphinscheduler-common/src/test/resources/sql/mysql_ddl.sql
b/dolphinscheduler-common/src/test/resources/sql/mysql_ddl.sql
new file mode 100644
index 0000000000..beadc63ca3
--- /dev/null
+++ b/dolphinscheduler-common/src/test/resources/sql/mysql_ddl.sql
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+ALTER TABLE t_ds_process_definition DROP tenant_id;
+ALTER TABLE t_ds_process_definition_log DROP tenant_id;
+ALTER TABLE t_ds_process_instance DROP tenant_id;
+
+SET FOREIGN_KEY_CHECKS = 0;
+-- auto detect by atlas, see more detail in
https://github.com/apache/dolphinscheduler/pull/14620
+ALTER TABLE `QRTZ_BLOB_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME`
varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_NAME` varchar(200) NOT NULL,
MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL;
+ALTER TABLE `QRTZ_CALENDARS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME`
varchar(120) NOT NULL, MODIFY COLUMN `CALENDAR_NAME` varchar(200) NOT NULL;
+ALTER TABLE `QRTZ_CRON_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME`
varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_NAME` varchar(200) NOT NULL,
MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL, MODIFY COLUMN
`CRON_EXPRESSION` varchar(120) NOT NULL, MODIFY COLUMN `TIME_ZONE_ID`
varchar(80) NULL;
+ALTER TABLE `QRTZ_FIRED_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME`
varchar(120) NOT NULL, MODIFY COLUMN `ENTRY_ID` varchar(200) NOT NULL, MODIFY
COLUMN `TRIGGER_NAME` varchar(200) NOT NULL, MODIFY COLUMN `TRIGGER_GROUP`
varchar(200) NOT NULL, MODIFY COLUMN `INSTANCE_NAME` varchar(200) NOT NULL,
MODIFY COLUMN `STATE` varchar(16) NOT NULL, MODIFY COLUMN `JOB_NAME`
varchar(200) NULL, MODIFY COLUMN `JOB_GROUP` varchar(200) NULL, MODIFY COLUMN
`IS_NONCONCURRENT` varchar(1) NULL, M [...]
+ALTER TABLE `QRTZ_JOB_DETAILS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME`
varchar(120) NOT NULL, MODIFY COLUMN `JOB_NAME` varchar(200) NOT NULL, MODIFY
COLUMN `JOB_GROUP` varchar(200) NOT NULL, MODIFY COLUMN `DESCRIPTION`
varchar(250) NULL, MODIFY COLUMN `JOB_CLASS_NAME` varchar(250) NOT NULL, MODIFY
COLUMN `IS_DURABLE` varchar(1) NOT NULL, MODIFY COLUMN `IS_NONCONCURRENT`
varchar(1) NOT NULL, MODIFY COLUMN `IS_UPDATE_DATA` varchar(1) NOT NULL, MODIFY
COLUMN `REQUESTS_RECOVERY` varchar [...]
+ALTER TABLE `QRTZ_LOCKS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME`
varchar(120) NOT NULL, MODIFY COLUMN `LOCK_NAME` varchar(40) NOT NULL;
+ALTER TABLE `QRTZ_PAUSED_TRIGGER_GRPS` COLLATE utf8_bin, MODIFY COLUMN
`SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_GROUP` varchar(200)
NOT NULL;
+ALTER TABLE `QRTZ_SCHEDULER_STATE` COLLATE utf8_bin, MODIFY COLUMN
`SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `INSTANCE_NAME` varchar(200)
NOT NULL;
+ALTER TABLE `QRTZ_SIMPLE_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN
`SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_NAME` varchar(200)
NOT NULL, MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL;
+ALTER TABLE `QRTZ_SIMPROP_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN
`SCHED_NAME` varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_NAME` varchar(200)
NOT NULL, MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL, MODIFY COLUMN
`STR_PROP_1` varchar(512) NULL, MODIFY COLUMN `STR_PROP_2` varchar(512) NULL,
MODIFY COLUMN `STR_PROP_3` varchar(512) NULL, MODIFY COLUMN `BOOL_PROP_1`
varchar(1) NULL, MODIFY COLUMN `BOOL_PROP_2` varchar(1) NULL;
+ALTER TABLE `QRTZ_TRIGGERS` COLLATE utf8_bin, MODIFY COLUMN `SCHED_NAME`
varchar(120) NOT NULL, MODIFY COLUMN `TRIGGER_NAME` varchar(200) NOT NULL,
MODIFY COLUMN `TRIGGER_GROUP` varchar(200) NOT NULL, MODIFY COLUMN `JOB_NAME`
varchar(200) NOT NULL, MODIFY COLUMN `JOB_GROUP` varchar(200) NOT NULL, MODIFY
COLUMN `DESCRIPTION` varchar(250) NULL, MODIFY COLUMN `TRIGGER_STATE`
varchar(16) NOT NULL, MODIFY COLUMN `TRIGGER_TYPE` varchar(8) NOT NULL, MODIFY
COLUMN `CALENDAR_NAME` varchar(200) NULL;
+ALTER TABLE `t_ds_plugin_define` AUTO_INCREMENT 2;
+ALTER TABLE `t_ds_process_instance` MODIFY COLUMN `state_history` text NULL
COMMENT 'state history desc';
+ALTER TABLE `t_ds_project` MODIFY COLUMN `description` varchar(255) NULL;
+ALTER TABLE `t_ds_task_group` MODIFY COLUMN `description` varchar(255) NULL;
+ALTER TABLE `t_ds_task_instance` MODIFY COLUMN `app_link` text NULL COMMENT
'yarn app id', MODIFY COLUMN `cache_key` varchar(200) NULL COMMENT 'cache_key',
MODIFY COLUMN `executor_name` varchar(64) NULL;
+ALTER TABLE `t_ds_worker_group` MODIFY COLUMN `description` text NULL COMMENT
'description';
+ALTER TABLE `t_ds_task_instance` MODIFY COLUMN `cache_key` varchar(200) NULL
COMMENT 'cache_key', MODIFY COLUMN `executor_name` varchar(64) NULL;
+ALTER TABLE `t_ds_fav_task` MODIFY COLUMN `id` bigint NOT NULL AUTO_INCREMENT
COMMENT 'id';
+ALTER TABLE `t_ds_task_instance` MODIFY COLUMN `cache_key` varchar(200) NULL
COMMENT 'cache_key', MODIFY COLUMN `executor_name` varchar(64) NULL;
+SET FOREIGN_KEY_CHECKS = 1;
diff --git a/dolphinscheduler-common/src/test/resources/sql/mysql_dml.sql
b/dolphinscheduler-common/src/test/resources/sql/mysql_dml.sql
new file mode 100644
index 0000000000..e5d97fab94
--- /dev/null
+++ b/dolphinscheduler-common/src/test/resources/sql/mysql_dml.sql
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+-- ############################# t_ds_tenant #############################
+drop PROCEDURE if EXISTS dolphin_t_ds_tenant_insert_default;
+delimiter d//
+CREATE PROCEDURE dolphin_t_ds_tenant_insert_default()
+BEGIN
+ IF
+NOT EXISTS(SELECT 1
+ FROM t_ds_tenant
+ WHERE id = -1)
+ THEN
+ INSERT INTO `t_ds_tenant` VALUES ('-1', 'default', 'default tenant',
'1', current_timestamp, current_timestamp);
+END IF;
+END;
+d//
+
+delimiter ;
+CALL dolphin_t_ds_tenant_insert_default();
+DROP PROCEDURE dolphin_t_ds_tenant_insert_default;
+
+-- tenant improvement
+UPDATE t_ds_schedules t1 JOIN t_ds_process_definition t2 ON
t1.process_definition_code = t2.code LEFT JOIN t_ds_tenant t3 ON t2.tenant_id =
t3.id SET t1.tenant_code = COALESCE(t3.tenant_code, 'default');
+UPDATE `t_ds_process_instance` SET `tenant_code` = 'default' WHERE
`tenant_code` IS NULL;
+
+-- data quality support choose database
+INSERT IGNORE INTO `t_ds_dq_rule_input_entry`
+(`id`, `field`, `type`, `title`, `value`, `options`, `placeholder`,
`option_source_type`, `value_type`, `input_type`, `is_show`, `can_edit`,
`is_emit`, `is_validate`, `create_time`, `update_time`)
+VALUES(30, 'src_database', 'select', '$t(src_database)', NULL, NULL, 'please
select source database', 0, 0, 0, 1, 1, 1, 1, current_timestamp,
current_timestamp);
+INSERT IGNORE INTO `t_ds_dq_rule_input_entry`
+(`id`, `field`, `type`, `title`, `value`, `options`, `placeholder`,
`option_source_type`, `value_type`, `input_type`, `is_show`, `can_edit`,
`is_emit`, `is_validate`, `create_time`, `update_time`)
+VALUES(31, 'target_database', 'select', '$t(target_database)', NULL, NULL,
'please select target database', 0, 0, 0, 1, 1, 1, 1, current_timestamp,
current_timestamp);
+
+INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
+(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`,
`update_time`)
+VALUES(151, 1, 30, NULL, 2, current_timestamp, current_timestamp);
+INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
+(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`,
`update_time`)
+VALUES(152, 2, 30, NULL, 2, current_timestamp, current_timestamp);
+INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
+(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`,
`update_time`)
+VALUES(153, 3, 30, NULL, 2, current_timestamp, current_timestamp);
+INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
+(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`,
`update_time`)
+VALUES(154, 4, 30, NULL, 2, current_timestamp, current_timestamp);
+INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
+(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`,
`update_time`)
+VALUES(155, 5, 30, NULL, 2, current_timestamp, current_timestamp);
+INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
+(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`,
`update_time`)
+VALUES(156, 6, 30, NULL, 2, current_timestamp, current_timestamp);
+INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
+(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`,
`update_time`)
+VALUES(157, 7, 30, NULL, 2, current_timestamp, current_timestamp);
+INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
+(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`,
`update_time`)
+VALUES(158, 8, 30, NULL, 2, current_timestamp, current_timestamp);
+INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
+(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`,
`update_time`)
+VALUES(159, 9, 30, NULL, 2, current_timestamp, current_timestamp);
+INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
+(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`,
`update_time`)
+VALUES(160, 10, 30, NULL, 2, current_timestamp, current_timestamp);
+INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
+(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`,
`update_time`)
+VALUES(161, 3, 31, NULL, 6, current_timestamp, current_timestamp);
+INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
+(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`,
`update_time`)
+VALUES(162, 4, 31, NULL, 7, current_timestamp, current_timestamp);
diff --git
a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/UpgradeDao.java
b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/UpgradeDao.java
index 343fbc6488..0154718780 100644
---
a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/UpgradeDao.java
+++
b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/UpgradeDao.java
@@ -17,13 +17,11 @@
package org.apache.dolphinscheduler.tools.datasource.dao;
-import org.apache.dolphinscheduler.common.utils.ScriptRunner;
+import org.apache.dolphinscheduler.common.sql.SqlScriptRunner;
import org.apache.dolphinscheduler.dao.upgrade.SchemaUtils;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.io.FileNotFoundException;
-import java.io.InputStreamReader;
-import java.io.Reader;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
@@ -33,9 +31,6 @@ import javax.sql.DataSource;
import lombok.extern.slf4j.Slf4j;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.core.io.Resource;
-
@Slf4j
public abstract class UpgradeDao {
@@ -63,17 +58,13 @@ public abstract class UpgradeDao {
* @param dbType db type
*/
private void runInitSql(DbType dbType) {
- String sqlFile = String.format("dolphinscheduler_%s.sql",
dbType.getDescp());
- Resource mysqlSQLFilePath = new ClassPathResource("sql/" + sqlFile);
- try (Connection conn = dataSource.getConnection()) {
- // Execute the dolphinscheduler_ddl.sql script to create the table
structure of dolphinscheduler
- ScriptRunner initScriptRunner = new ScriptRunner(conn, true, true);
- try (Reader initSqlReader = new
InputStreamReader(mysqlSQLFilePath.getInputStream())) {
- initScriptRunner.runScript(initSqlReader);
- }
- } catch (Exception e) {
- log.error("Execute init sql file: {} error", sqlFile, e);
- throw new RuntimeException(String.format("Execute init sql file:
%s error", sqlFile), e);
+ String sqlFilePath = String.format("sql/dolphinscheduler_%s.sql",
dbType.getDescp());
+ SqlScriptRunner sqlScriptRunner = new SqlScriptRunner(dataSource,
sqlFilePath);
+ try {
+ sqlScriptRunner.execute();
+ log.info("Success execute the sql initialize file: {}",
sqlFilePath);
+ } catch (Exception ex) {
+ throw new RuntimeException("Execute initialize sql file: " +
sqlFilePath + " error", ex);
}
}
@@ -120,14 +111,13 @@ public abstract class UpgradeDao {
private void upgradeDolphinSchedulerDML(String schemaDir, String
scriptFile) {
String schemaVersion = schemaDir.split("_")[0];
- Resource sqlFilePath = new ClassPathResource(
- String.format("sql/upgrade/%s/%s/%s", schemaDir,
getDbType().name().toLowerCase(), scriptFile));
- try (Connection conn = dataSource.getConnection()) {
- conn.setAutoCommit(false);
+ String sqlFilePath =
+ String.format("sql/upgrade/%s/%s/%s", schemaDir,
getDbType().name().toLowerCase(), scriptFile);
+ try {
// Execute the upgraded dolphinscheduler dml
- ScriptRunner scriptRunner = new ScriptRunner(conn, false, true);
- try (Reader sqlReader = new
InputStreamReader(sqlFilePath.getInputStream())) {
- scriptRunner.runScript(sqlReader);
+ SqlScriptRunner sqlScriptRunner = new SqlScriptRunner(dataSource,
sqlFilePath);
+ sqlScriptRunner.execute();
+ try (Connection connection = dataSource.getConnection()) {
String upgradeSQL;
if (isExistsTable(T_VERSION_NAME)) {
// Change version in the version table to the new version
@@ -138,11 +128,10 @@ public abstract class UpgradeDao {
} else {
throw new RuntimeException("The version table does not
exist");
}
- try (PreparedStatement pstmt =
conn.prepareStatement(upgradeSQL)) {
+ try (PreparedStatement pstmt =
connection.prepareStatement(upgradeSQL)) {
pstmt.setString(1, schemaVersion);
pstmt.executeUpdate();
}
- conn.commit();
}
log.info("Success execute the dml file, schemaDir: {}, ddlScript:
{}", schemaDir, scriptFile);
} catch (FileNotFoundException e) {
@@ -161,15 +150,12 @@ public abstract class UpgradeDao {
* @param schemaDir schemaDir
*/
public void upgradeDolphinSchedulerDDL(String schemaDir, String
scriptFile) {
- Resource sqlFilePath = new ClassPathResource(
- String.format("sql/upgrade/%s/%s/%s", schemaDir,
getDbType().name().toLowerCase(), scriptFile));
- try (Connection conn = dataSource.getConnection()) {
- conn.setAutoCommit(true);
+ String sqlFilePath =
+ String.format("sql/upgrade/%s/%s/%s", schemaDir,
getDbType().name().toLowerCase(), scriptFile);
+ SqlScriptRunner sqlScriptRunner = new SqlScriptRunner(dataSource,
sqlFilePath);
+ try {
// Execute the dolphinscheduler ddl.sql for the upgrade
- ScriptRunner scriptRunner = new ScriptRunner(conn, true, true);
- try (Reader sqlReader = new
InputStreamReader(sqlFilePath.getInputStream())) {
- scriptRunner.runScript(sqlReader);
- }
+ sqlScriptRunner.execute();
log.info("Success execute the ddl file, schemaDir: {}, ddlScript:
{}", schemaDir, scriptFile);
} catch (FileNotFoundException e) {
log.error("Cannot find the DDL file, schemaDir: {}, ddlScript:
{}", schemaDir, scriptFile, e);
