This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d4142171737d5ff0e7d0e6d78fd35ea866c5bf61
Author: Jibing-Li <64681310+jibing...@users.noreply.github.com>
AuthorDate: Thu Aug 1 16:22:58 2024 +0800
[improvement](statisitcs)Disable external catalog auto analyze by default.
(#38674)
Disable external catalog auto analyze by default.
---
.../apache/doris/datasource/ExternalCatalog.java | 4 +--
.../doris/statistics/util/StatisticsUtil.java | 8 +++++
.../doris/statistics/util/StatisticsUtilTest.java | 42 ++++++++++++++++++++--
3 files changed, 50 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index a425c782fe3..265e4f35232 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -835,8 +835,8 @@ public abstract class ExternalCatalog
@Override
public boolean enableAutoAnalyze() {
- // By default, external catalog disables auto analyze, uses could set
catalog property to enable it:
- // "enable.auto.analyze" = true
+ // By default, external catalog disables auto analyze, users could set
catalog property to enable it:
+ // "enable.auto.analyze" = "true"
Map<String, String> properties = catalogProperty.getProperties();
boolean ret = false;
if (properties.containsKey(ENABLE_AUTO_ANALYZE)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index 76da12f20c8..5947fcc0e60 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -980,6 +980,14 @@ public class StatisticsUtil {
if (column == null) {
return false;
}
+ try {
+ if (!table.getDatabase().getCatalog().enableAutoAnalyze()) {
+ return false;
+ }
+ } catch (Throwable t) {
+ LOG.warn("Failed to get catalog property. {}", t.getMessage());
+ return false;
+ }
AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
TableStatsMeta tableStatsStatus =
manager.findTableStatsStatus(table.getId());
// Table never been analyzed, need analyze.
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
index ebe86ce9d61..90abe3fca17 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
@@ -18,12 +18,18 @@
package org.apache.doris.statistics.util;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.ExternalTable;
+import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable.DLAType;
import org.apache.doris.datasource.jdbc.JdbcExternalTable;
@@ -164,18 +170,50 @@ class StatisticsUtilTest {
}
@Test
- void testNeedAnalyzeColumn() {
+ void testNeedAnalyzeColumn() throws DdlException {
Column column = new Column("testColumn", PrimitiveType.INT);
List<Column> schema = new ArrayList<>();
schema.add(column);
OlapTable table = new OlapTable(200, "testTable", schema, null, null,
null);
- // Test table stats meta is null.
+ Database db = new Database(111, "TestDb");
+ ExternalCatalog externalCatalog = new HMSExternalCatalog();
+
+ // Test get database/catalog exception
+ new MockUp<OlapTable>() {
+ @Mock
+ public DatabaseIf getDatabase() {
+ throw new RuntimeException();
+ }
+ };
+ Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(table,
Pair.of("index", column.getName())));
+
+ // Test auto analyze disabled.
+ new MockUp<OlapTable>() {
+ @Mock
+ public DatabaseIf getDatabase() {
+ return db;
+ }
+ };
+ new MockUp<Database>() {
+ @Mock
+ public CatalogIf getCatalog() {
+ return externalCatalog;
+ }
+ };
+ Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(table,
Pair.of("index", column.getName())));
+
+ // Test auto analyze enabled.
new MockUp<AnalysisManager>() {
@Mock
public TableStatsMeta findTableStatsStatus(long tblId) {
return null;
}
};
+
externalCatalog.getCatalogProperty().addProperty(ExternalCatalog.ENABLE_AUTO_ANALYZE,
"true");
+ Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table,
Pair.of("index", column.getName())));
+
+
+ // Test table stats meta is null.
Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table,
Pair.of("index", column.getName())));
// Test user injected flag is set.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
