This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5d52162484b [Test](statistics) Add test cases for external table
statistics (#26511)
5d52162484b is described below
commit 5d52162484b52524ec471611425ab22b6ee30212
Author: Jibing-Li <[email protected]>
AuthorDate: Thu Nov 9 12:12:29 2023 +0800
[Test](statistics) Add test cases for external table statistics (#26511)
1. Test for close and open auto collection for external catalog.
2. Test for analyze table table_name (column) and whole table.
---
.../apache/doris/datasource/ExternalCatalog.java | 3 +
.../doris/datasource/HMSExternalCatalog.java | 4 ++
.../doris/datasource/ExternalCatalogTest.java | 50 +++++++++++++++
.../hive/test_hive_statistics_p0.groovy | 72 +++++++++++++++++++++-
4 files changed, 128 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index e764fdd356f..7393a75bac7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -104,6 +104,9 @@ public abstract class ExternalCatalog
private ExternalSchemaCache schemaCache;
private String comment;
+ public ExternalCatalog() {
+ }
+
public ExternalCatalog(long catalogId, String name, InitCatalogLog.Type
logType, String comment) {
this.id = catalogId;
this.name = name;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
index 7ca0ac0cff4..408504f5cc2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
@@ -70,6 +70,10 @@ public class HMSExternalCatalog extends ExternalCatalog {
// 0 means file cache is disabled; >0 means file cache with ttl;
public static final int FILE_META_CACHE_TTL_DISABLE_CACHE = 0;
+ public HMSExternalCatalog() {
+ catalogProperty = new CatalogProperty(null, null);
+ }
+
/**
* Default constructor for HMSExternalCatalog.
*/
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
new file mode 100644
index 00000000000..34f5352597f
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource;
+
+import org.apache.doris.utframe.TestWithFeService;
+
+import com.google.common.collect.Maps;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+
+public class ExternalCatalogTest extends TestWithFeService {
+
+ @Test
+ public void testExternalCatalogAutoAnalyze() throws Exception {
+ HMSExternalCatalog catalog = new HMSExternalCatalog();
+ Assertions.assertFalse(catalog.enableAutoAnalyze());
+
+ HashMap<String, String> prop = Maps.newHashMap();
+ prop.put(ExternalCatalog.ENABLE_AUTO_ANALYZE, "false");
+ catalog.modifyCatalogProps(prop);
+ Assertions.assertFalse(catalog.enableAutoAnalyze());
+
+ prop = Maps.newHashMap();
+ prop.put(ExternalCatalog.ENABLE_AUTO_ANALYZE, "true");
+ catalog.modifyCatalogProps(prop);
+ Assertions.assertTrue(catalog.enableAutoAnalyze());
+
+ prop = Maps.newHashMap();
+ prop.put(ExternalCatalog.ENABLE_AUTO_ANALYZE, "TRUE");
+ catalog.modifyCatalogProps(prop);
+ Assertions.assertTrue(catalog.enableAutoAnalyze());
+ }
+}
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_statistics_p0.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_statistics_p0.groovy
index 38db4379a90..ae041a67b3a 100644
---
a/regression-test/suites/external_table_p0/hive/test_hive_statistics_p0.groovy
+++
b/regression-test/suites/external_table_p0/hive/test_hive_statistics_p0.groovy
@@ -30,7 +30,33 @@ suite("test_hive_statistics_p0",
"all_types,p0,external,hive,external_docker,ext
);"""
sql """use `${catalog_name}`.`stats_test`"""
sql """analyze database stats_test with sync"""
- def result = sql """show column stats stats_test1(id);"""
+
+
+ def result = sql """show catalog ${catalog_name}"""
+ for (int i = 0; i < result.size(); i++) {
+ assertNotEquals("enable.auto.analyze", result[i][0]);
+ }
+ sql """alter catalog ${catalog_name} set properties
("enable.auto.analyze" = "true");"""
+ result = sql """show catalog ${catalog_name}"""
+ def flag = false;
+ for (int i = 0; i < result.size(); i++) {
+ if
("enable.auto.analyze".equalsIgnoreCase((String)result[i][0])
+ && "true".equalsIgnoreCase((String)result[i][1])) {
+ flag = true;
+ logger.info("enable.auto.analyze has been set to true")
+ }
+ }
+ assertTrue(flag);
+ sql """alter catalog ${catalog_name} set properties
("enable.auto.analyze" = "false");"""
+ result = sql """show catalog ${catalog_name}"""
+ for (int i = 0; i < result.size(); i++) {
+ if
("enable.auto.analyze".equalsIgnoreCase((String)result[i][0])) {
+ assertNotEquals("true",
((String)result[i][1]).toLowerCase());
+ logger.info("enable.auto.analyze has been set back to
false")
+ }
+ }
+
+ result = sql """show column stats stats_test1(id);"""
assertEquals(1, result.size())
assertEquals("id", result[0][0])
assertEquals("3.0", result[0][1])
@@ -74,8 +100,52 @@ suite("test_hive_statistics_p0",
"all_types,p0,external,hive,external_docker,ext
assertEquals("\'*\'", result[0][6])
assertEquals("\';\'", result[0][7])
+ sql """drop catalog if exists ${catalog_name}"""
+
+ sql """create catalog if not exists ${catalog_name} properties (
+ "type"="hms",
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}'
+ );"""
+ sql """use `${catalog_name}`.`stats_test`"""
+ sql """analyze table stats_test1(value) with sync"""
+ result = sql """show column stats stats_test1(value);"""
+ assertEquals(1, result.size())
+ assertEquals("value", result[0][0])
+ assertEquals("3.0", result[0][1])
+ assertEquals("3.0", result[0][2])
+ assertEquals("0.0", result[0][3])
+ assertEquals("15.0", result[0][4])
+ assertEquals("5.0", result[0][5])
+ assertEquals("\'name1\'" , result[0][6])
+ assertEquals("\'name3\'" , result[0][7])
+
+ result = sql """show column stats stats_test1(id);"""
+ assertEquals(0, result.size())
+ sql """analyze table stats_test2 with sync;"""
+ result = sql """show column stats stats_test2(id);"""
+ assertEquals(1, result.size())
+ assertEquals("id", result[0][0])
+ assertEquals("2.0", result[0][1])
+ assertEquals("2.0", result[0][2])
+ assertEquals("0.0", result[0][3])
+ assertEquals("8.0", result[0][4])
+ assertEquals("4.0", result[0][5])
+ assertEquals("1", result[0][6])
+ assertEquals("2", result[0][7])
+
+ result = sql """show column stats stats_test2(value);"""
+ assertEquals(1, result.size())
+ assertEquals("value", result[0][0])
+ assertEquals("2.0", result[0][1])
+ assertEquals("2.0", result[0][2])
+ assertEquals("0.0", result[0][3])
+ assertEquals("2.0", result[0][4])
+ assertEquals("1.0", result[0][5])
+ assertEquals("\'*\'", result[0][6])
+ assertEquals("\';\'", result[0][7])
sql """drop catalog if exists ${catalog_name}"""
+
} finally {
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
