This is an automated email from the ASF dual-hosted git repository.
zhangstar333 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e6fe5ccbb3a [opt](analyze) support iceberg external table in analyze
(#59473)
e6fe5ccbb3a is described below
commit e6fe5ccbb3a32958e382256e769b1dc06953007b
Author: zhangstar333 <[email protected]>
AuthorDate: Tue Jan 6 10:10:10 2026 +0800
[opt](analyze) support iceberg external table in analyze (#59473)
### What problem does this PR solve?
Problem Summary:
1. support iceberg external table in analyze
2. fix SHOW QUEUED ANALYZE JOBS could running normal.
```
mysql> SHOW QUEUED ANALYZE JOBS;
+---------------------------+---------------+-------------------------+---------------------------------------------------------------+----------+
| catalog_name | db_name | tbl_name |
col_list | priority |
+---------------------------+---------------+-------------------------+---------------------------------------------------------------+----------+
| test_iceberg_with_mapping | multi_catalog | orc_partitioned_columns |
orc_partitioned_columns:t_int,orc_partitioned_columns:t_float | HIGH |
+---------------------------+---------------+-------------------------+---------------------------------------------------------------+----------+
```
---
.../apache/doris/statistics/AnalysisManager.java | 27 +++++++++++++++++++---
.../doris/statistics/StatisticsAutoCollector.java | 9 ++++++--
.../doris/statistics/util/StatisticsUtil.java | 15 +++++++-----
3 files changed, 40 insertions(+), 11 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index dae18a9a15e..73c8817c20a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -594,9 +594,7 @@ public class AnalysisManager implements Writable {
for (Entry<TableNameInfo, Set<Pair<String, String>>> entry :
jobMap.entrySet()) {
TableNameInfo table = entry.getKey();
if (tableNameInfo == null
- || tableNameInfo.getCtl() == null &&
tableNameInfo.getDb() == null
- && tableNameInfo.getTbl() == null
- || tableNameInfo.equals(table)) {
+ || matchesFilter(tableNameInfo, table)) {
result.add(new AutoAnalysisPendingJob(table.getCtl(),
table.getDb(), table.getTbl(), entry.getValue(),
priority));
}
@@ -605,6 +603,29 @@ public class AnalysisManager implements Writable {
return result;
}
+ private boolean matchesFilter(TableNameInfo filter, TableNameInfo target) {
+ if (StringUtils.isEmpty(filter.getCtl())
+ && StringUtils.isEmpty(filter.getDb())
+ && StringUtils.isEmpty(filter.getTbl())) {
+ return true;
+ }
+
+ if (!StringUtils.isEmpty(filter.getCtl())
+ && !filter.getCtl().equals(target.getCtl())) {
+ return false;
+ }
+ if (!StringUtils.isEmpty(filter.getDb())
+ && !filter.getDb().equals(target.getDb())) {
+ return false;
+ }
+ if (!StringUtils.isEmpty(filter.getTbl())
+ && !filter.getTbl().equals(target.getTbl())) {
+ return false;
+ }
+
+ return true;
+ }
+
public List<AnalysisInfo> findAnalysisJobs(String state, String ctl,
String db,
String table, long jobId, boolean isAuto) {
TableIf tbl = null;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index b0550fe78d6..40b1a7022c8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -25,7 +25,9 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.MasterDaemon;
+import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
import org.apache.doris.info.TableNameInfo;
import org.apache.doris.persist.TableStatsDeletionLog;
import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
@@ -148,6 +150,9 @@ public class StatisticsAutoCollector extends MasterDaemon {
if (StatisticsUtil.enablePartitionAnalyze() &&
table.isPartitionedTable()) {
analysisMethod = AnalysisMethod.FULL;
}
+ if (table instanceof ExternalTable) { // External table only support
full analyze now
+ analysisMethod = AnalysisMethod.FULL;
+ }
boolean isSampleAnalyze = analysisMethod.equals(AnalysisMethod.SAMPLE);
OlapTable olapTable = table instanceof OlapTable ? (OlapTable) table :
null;
AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
@@ -229,9 +234,9 @@ public class StatisticsAutoCollector extends MasterDaemon {
if (tableIf == null) {
return false;
}
- return tableIf instanceof OlapTable
+ return tableIf instanceof OlapTable || tableIf instanceof
IcebergExternalTable
|| tableIf instanceof HMSExternalTable
- && ((HMSExternalTable)
tableIf).getDlaType().equals(HMSExternalTable.DLAType.HIVE);
+ && ((HMSExternalTable)
tableIf).getDlaType().equals(HMSExternalTable.DLAType.HIVE);
}
protected AnalysisInfo createAnalyzeJobForTbl(TableIf table,
Set<Pair<String, String>> jobColumns,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index 873ca0d2c99..9cf4d34f766 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -55,6 +55,7 @@ import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable.DLAType;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
import org.apache.doris.info.TableNameInfo;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.IPv4Literal;
@@ -1130,15 +1131,17 @@ public class StatisticsUtil {
// 3. Check partition
return needAnalyzePartition(olapTable, tableStatsStatus,
columnStatsMeta);
} else {
- // Now, we only support Hive external table auto analyze.
- if (!(table instanceof HMSExternalTable)) {
+ if (!(table instanceof HMSExternalTable || (table instanceof
IcebergExternalTable))) {
return false;
}
- HMSExternalTable hmsTable = (HMSExternalTable) table;
- if (!hmsTable.getDlaType().equals(DLAType.HIVE)) {
- return false;
+ if (table instanceof HMSExternalTable) {
+ HMSExternalTable hmsTable = (HMSExternalTable) table;
+ if (!hmsTable.getDlaType().equals(DLAType.HIVE) &&
!hmsTable.getDlaType().equals(DLAType.ICEBERG)) {
+ return false;
+ }
}
- // External is hard to calculate change rate, use time interval to
control analyze frequency.
+ // External is hard to calculate change rate, use time interval to
control
+ // analyze frequency.
return System.currentTimeMillis()
- tableStatsStatus.lastAnalyzeTime >
StatisticsUtil.getExternalTableAutoAnalyzeIntervalInMillis();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
