This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2bce4db81a [Enchancement](mysql-compatable) add regression-test for
MySQLdump #18208
2bce4db81a is described below
commit 2bce4db81adb5e0f2bad670a9d52082687887087
Author: WenYao <[email protected]>
AuthorDate: Mon Apr 3 09:49:07 2023 +0800
[Enchancement](mysql-compatable) add regression-test for MySQLdump #18208
add regression-test for like this:
mysqldump -h127.0.0.1 -P9030 -uroot --no-tablespaces --databases >
/backup/mysqldump/test.db
To prevent errors Unknown table 'column_statistics' in information_schema
(1109), the table information_schema.column_statistics was added.
---
.../org/apache/doris/analysis/SchemaTableType.java | 1 +
.../java/org/apache/doris/catalog/SchemaTable.java | 7 ++
gensrc/thrift/Descriptors.thrift | 3 +-
regression-test/data/mysqldump_p0/data.csv | 25 +++++
.../data/mysqldump_p0/test_mysqldump.out | 47 +++++++++
.../suites/mysqldump_p0/test_mysqldump.groovy | 117 +++++++++++++++++++++
6 files changed, 199 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java
index 3e2615d8e0..81fc050239 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java
@@ -53,6 +53,7 @@ public enum SchemaTableType {
SCH_SESSION_VARIABLES("SESSION_VARIABLES", "SESSION_VARIABLES",
TSchemaTableType.SCH_SESSION_VARIABLES),
SCH_STATISTICS("STATISTICS", "STATISTICS",
TSchemaTableType.SCH_STATISTICS),
+ SCH_COLUMN_STATISTICS("COLUMN_STATISTICS", "COLUMN_STATISTICS",
TSchemaTableType.SCH_COLUMN_STATISTICS),
SCH_STATUS("STATUS", "STATUS", TSchemaTableType.SCH_STATUS),
SCH_TABLES("TABLES", "TABLES", TSchemaTableType.SCH_TABLES),
SCH_TABLE_CONSTRAINTS("TABLE_CONSTRAINTS", "TABLE_CONSTRAINTS",
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
index 4f6ddb032e..af8cc101b2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
@@ -243,6 +243,13 @@ public class SchemaTable extends Table {
.column("COMMENT", ScalarType.createVarchar(16))
// for datagrip
.column("INDEX_COMMENT",
ScalarType.createVarchar(1024)).build()))
+ // Compatible with mysql for mysqldump
+ .put("column_statistics",
+ new SchemaTable(SystemIdGenerator.getNextId(),
"column_statistics", TableType.SCHEMA,
+ builder().column("SCHEMA_NAME",
ScalarType.createVarchar(64))
+ .column("TABLE_NAME", ScalarType.createVarchar(64))
+ .column("COLUMN_NAME",
ScalarType.createVarchar(64))
+ .column("HISTOGRAM",
ScalarType.createJsonbType()).build()))
.put("files",
new SchemaTable(SystemIdGenerator.getNextId(), "files",
TableType.SCHEMA,
builder().column("FILE_ID",
ScalarType.createType(PrimitiveType.BIGINT))
diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift
index 8d7c646eec..9a5043972c 100644
--- a/gensrc/thrift/Descriptors.thrift
+++ b/gensrc/thrift/Descriptors.thrift
@@ -111,7 +111,8 @@ enum TSchemaTableType {
SCH_VIEWS,
SCH_INVALID,
SCH_ROWSETS,
- SCH_BACKENDS
+ SCH_BACKENDS,
+ SCH_COLUMN_STATISTICS
}
enum THdfsCompression {
diff --git a/regression-test/data/mysqldump_p0/data.csv
b/regression-test/data/mysqldump_p0/data.csv
new file mode 100644
index 0000000000..2ad487e864
--- /dev/null
+++ b/regression-test/data/mysqldump_p0/data.csv
@@ -0,0 +1,25 @@
+0,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+1,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+2,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+3,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+4,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+5,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+6,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+7,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+8,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+9,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+10,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+11,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+12,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+13,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+14,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+15,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+16,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+17,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+18,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+19,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+20,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+21,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+22,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+23,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
+24,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59
\ No newline at end of file
diff --git a/regression-test/data/mysqldump_p0/test_mysqldump.out
b/regression-test/data/mysqldump_p0/test_mysqldump.out
new file mode 100644
index 0000000000..2dda4e1b0c
--- /dev/null
+++ b/regression-test/data/mysqldump_p0/test_mysqldump.out
@@ -0,0 +1,47 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+2
+
+-- !sql --
+25
+
+-- !sql --
+0 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+1 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+2 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+3 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+4 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+
+-- !sql --
+25
+
+-- !sql --
+0 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+1 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+2 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+3 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+4 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+
+-- !sql --
+2
+
+-- !sql --
+25
+
+-- !sql --
+0 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+1 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+2 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+3 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+4 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+
+-- !sql --
+25
+
+-- !sql --
+0 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+1 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+2 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+3 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+4 2 3 4 5 6.6 7.7 9 abc def
ghiaaaaaa 2020-10-10 2020-10-10T11:12:59
+
diff --git a/regression-test/suites/mysqldump_p0/test_mysqldump.groovy
b/regression-test/suites/mysqldump_p0/test_mysqldump.groovy
new file mode 100644
index 0000000000..332409d07e
--- /dev/null
+++ b/regression-test/suites/mysqldump_p0/test_mysqldump.groovy
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_mysqldump") { suite ->
+
+ def executeMySQLCommand = { String command ->
+ def cmds = ["/bin/bash", "-c", command]
+ logger.info("Execute: ${cmds}".toString())
+ Process p = cmds.execute()
+
+ def errMsg = new StringBuilder()
+ def msg = new StringBuilder()
+ p.waitForProcessOutput(msg, errMsg)
+
+ assert errMsg.length() == 0: "error occurred!" + errMsg
+ assert p.exitValue() == 0
+ }
+
+ def dbName = context.config.getDbNameByFile(context.file)
+ def tablePrefix = "test_mysqldump_table_"
+
+ for (int i = 1; i <= 2; ++i) {
+ def tableName = tablePrefix + i.toString();
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ }
+
+ for (int i = 1; i <= 2; ++i) {
+ def tableName = tablePrefix + i.toString();
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `k1` int(11) NULL,
+ `k2` tinyint(4) NULL,
+ `k3` smallint(6) NULL,
+ `k4` bigint(20) NULL,
+ `k5` largeint(40) NULL,
+ `k6` float NULL,
+ `k7` double NULL,
+ `k8` decimal(9, 0) NULL,
+ `k9` char(10) NULL,
+ `k10` varchar(1024) NULL,
+ `k11` text NULL,
+ `k12` date NULL,
+ `k13` datetime NULL
+ ) ENGINE=OLAP
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 3
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+ // load all columns
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', ','
+
+ file 'data.csv'
+ time 10000 // limit inflight 10s
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(25, json.NumberTotalRows)
+ assertEquals(0, json.NumberFilteredRows)
+ }
+ }
+ }
+
+ qt_sql "select count(*) from information_schema.tables where table_schema
= '${dbName}'"
+ for (int i = 1; i <= 2; ++i) {
+ def tableName = tablePrefix + i.toString();
+ qt_sql "select count(*) from ${tableName}"
+ qt_sql "select * from ${tableName} order by k1 limit 5"
+ }
+
+ // use mysqldump
+ String jdbcUrlConfig = context.config.jdbcUrl;
+ String tempString =
jdbcUrlConfig.substring(jdbcUrlConfig.indexOf("jdbc:mysql://") + 13);
+ String mysqlHost = tempString.substring(0, tempString.indexOf(":"));
+ String mysqlPort = tempString.substring(tempString.indexOf(":") + 1,
tempString.indexOf("/"));
+ def filePath = "/tmp/mysql_${dbName}.db"
+ String cmdMySQLdump = "mysqldump -uroot -h" + mysqlHost + " -P" +
mysqlPort + " --no-tablespaces --databases ${dbName} > ${filePath}";
+ executeMySQLCommand(cmdMySQLdump);
+
+ // restore
+ for (int i = 1; i <= 2; ++i) {
+ def tableName = tablePrefix + i.toString();
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ }
+ sql "DROP DATABASE IF EXISTS ${dbName}"
+
+ String cmdRestore = "mysql -uroot -h" + mysqlHost + " -P" + mysqlPort + "
< ${filePath}";
+ executeMySQLCommand(cmdRestore);
+ qt_sql "select count(*) from information_schema.tables where table_schema
= '${dbName}'"
+ for (int i = 1; i <= 2; ++i) {
+ def tableName = tablePrefix + i.toString();
+ qt_sql "select count(*) from ${tableName}"
+ qt_sql "select * from ${tableName} order by k1 limit 5"
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
