This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-4.0-preview
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0-preview by this
push:
new 2f3be24c5cf Revert "[fix](statistics) Use column update rows to decide
min/max stats are valid or not (#34263)"
2f3be24c5cf is described below
commit 2f3be24c5cfe6e322ded1dbaebfc706c7b62777d
Author: Yongqiang YANG <[email protected]>
AuthorDate: Wed May 8 11:22:32 2024 +0800
Revert "[fix](statistics) Use column update rows to decide min/max stats
are valid or not (#34263)"
This reverts commit aeafc99a7b193db40f369cd537ad4b7d3b517400.
---
.../doris/nereids/stats/StatsCalculator.java | 26 ++++++----------------
1 file changed, 7 insertions(+), 19 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index 96d08e6f3a1..34248d5a55a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -19,7 +19,6 @@ package org.apache.doris.nereids.stats;
import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.catalog.Env;
-import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
@@ -125,7 +124,6 @@ import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.util.PlanUtils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.statistics.AnalysisManager;
-import org.apache.doris.statistics.ColStatsMeta;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.ColumnStatisticBuilder;
import org.apache.doris.statistics.Histogram;
@@ -766,10 +764,10 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
Set<SlotReference> slotSet = slotSetBuilder.build();
Map<Expression, ColumnStatisticBuilder> columnStatisticBuilderMap =
new HashMap<>();
TableIf table = catalogRelation.getTable();
- boolean isOlapTable = table instanceof OlapTable;
AnalysisManager analysisManager =
Env.getCurrentEnv().getAnalysisManager();
TableStatsMeta tableMeta =
analysisManager.findTableStatsStatus(table.getId());
- long tableUpdatedRows = tableMeta == null ? 0 :
tableMeta.updatedRows.get();
+ // rows newly updated after last analyze
+ long deltaRowCount = tableMeta == null ? 0 :
tableMeta.updatedRows.get();
double rowCount = catalogRelation.getTable().getRowCountForNereids();
boolean hasUnknownCol = false;
long idxId = -1;
@@ -779,6 +777,10 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
idxId = olapScan.getSelectedIndexId();
}
}
+ if (deltaRowCount > 0 && LOG.isDebugEnabled()) {
+ LOG.debug("{} is partially analyzed, clear min/max values in
column stats",
+ catalogRelation.getTable().getName());
+ }
for (SlotReference slotReference : slotSet) {
String colName = slotReference.getColumn().isPresent()
? slotReference.getColumn().get().getName()
@@ -788,13 +790,6 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
if (colName == null) {
throw new RuntimeException(String.format("Invalid slot: %s",
slotReference.getExprId()));
}
- long deltaRowCount = 0;
- if (isOlapTable) {
- OlapTable olapTable = (OlapTable) table;
- ColStatsMeta colMeta = tableMeta == null ? null :
tableMeta.findColumnStatsMeta(
- olapTable.getIndexNameById(idxId == -1 ?
olapTable.getBaseIndexId() : idxId), colName);
- deltaRowCount = colMeta == null ? 0 : tableUpdatedRows -
colMeta.updatedRows;
- }
ColumnStatistic cache;
if (!FeConstants.enableInternalSchemaDb
|| shouldIgnoreThisCol) {
@@ -812,20 +807,13 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
hasUnknownCol = true;
}
if (ConnectContext.get() != null &&
ConnectContext.get().getSessionVariable().enableStats) {
- // deltaRowCount > 0 indicates that
- // new data is loaded to the table after this column was
analyzed last time.
- // In this case, need to eliminate min/max value for this
column.
if (deltaRowCount > 0) {
// clear min-max to avoid error estimation
// for example, after yesterday data loaded, user send
query about yesterday immediately.
// since yesterday data are not analyzed, the max date is
before yesterday, and hence optimizer
// estimates the filter result is zero
colStatsBuilder.setMinExpr(null).setMinValue(Double.NEGATIVE_INFINITY)
-
.setMaxExpr(null).setMaxValue(Double.POSITIVE_INFINITY);
- if (LOG.isDebugEnabled()) {
- LOG.debug("{}.{} is partially analyzed, clear min/max
values in column stats",
- table.getName(), colName);
- }
+
.setMaxExpr(null).setMaxValue(Double.POSITIVE_INFINITY);
}
columnStatisticBuilderMap.put(slotReference, colStatsBuilder);
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
