This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c4bade71c8 [refactor](nereids) remove ColumnStatistics.UNKNOWN from
StatsDerive (#14343)
c4bade71c8 is described below
commit c4bade71c8959afb4d8b0cb4ae21e832abc7d759
Author: minghong <[email protected]>
AuthorDate: Fri Nov 18 23:40:00 2022 +0800
[refactor](nereids) remove ColumnStatistics.UNKNOWN from StatsDerive
(#14343)
ColumnStatistics.UNKNOWN can be replaced by ColumnStatistics.DEFAULT
---
.../apache/doris/nereids/stats/ExpressionEstimation.java | 14 +++++++-------
.../org/apache/doris/nereids/stats/FilterEstimation.java | 6 +++---
.../org/apache/doris/nereids/stats/StatsCalculator.java | 2 +-
.../org/apache/doris/nereids/stats/StatsCalculatorV2.java | 3 ---
.../java/org/apache/doris/statistics/ColumnStatistic.java | 10 +++-------
.../apache/doris/statistics/ColumnStatisticBuilder.java | 2 +-
.../java/org/apache/doris/statistics/StatisticsCache.java | 6 +++---
.../org/apache/doris/statistics/StatisticsRepository.java | 2 +-
.../test/java/org/apache/doris/statistics/CacheTest.java | 6 +++---
9 files changed, 22 insertions(+), 29 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
index 269b299ecb..675632c681 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
@@ -161,7 +161,7 @@ public class ExpressionEstimation extends
ExpressionVisitor<ColumnStatistic, Sta
.setNumNulls(numNulls).setDataSize(binaryArithmetic.getDataType().width()).setMinValue(min)
.setMaxValue(max).setSelectivity(1.0).setMaxExpr(null).setMinExpr(null).build();
}
- return ColumnStatistic.UNKNOWN;
+ return ColumnStatistic.DEFAULT;
}
private double noneZeroDivisor(double d) {
@@ -172,8 +172,8 @@ public class ExpressionEstimation extends
ExpressionVisitor<ColumnStatistic, Sta
public ColumnStatistic visitMin(Min min, StatsDeriveResult context) {
Expression child = min.child();
ColumnStatistic columnStat = child.accept(this, context);
- if (columnStat == ColumnStatistic.UNKNOWN) {
- return ColumnStatistic.UNKNOWN;
+ if (columnStat == ColumnStatistic.DEFAULT) {
+ return ColumnStatistic.DEFAULT;
}
/*
we keep columnStat.min and columnStat.max, but set ndv=1.
@@ -190,8 +190,8 @@ public class ExpressionEstimation extends
ExpressionVisitor<ColumnStatistic, Sta
public ColumnStatistic visitMax(Max max, StatsDeriveResult context) {
Expression child = max.child();
ColumnStatistic columnStat = child.accept(this, context);
- if (columnStat == ColumnStatistic.UNKNOWN) {
- return ColumnStatistic.UNKNOWN;
+ if (columnStat == ColumnStatistic.DEFAULT) {
+ return ColumnStatistic.DEFAULT;
}
/*
we keep columnStat.min and columnStat.max, but set ndv=1.
@@ -210,8 +210,8 @@ public class ExpressionEstimation extends
ExpressionVisitor<ColumnStatistic, Sta
}
Expression child = count.child(0);
ColumnStatistic columnStat = child.accept(this, context);
- if (columnStat == ColumnStatistic.UNKNOWN) {
- return ColumnStatistic.UNKNOWN;
+ if (columnStat == ColumnStatistic.DEFAULT) {
+ return ColumnStatistic.DEFAULT;
}
double expectedValue = context.getRowCount() - columnStat.numNulls;
double width = (double) count.getDataType().width();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
index e9f2debee0..76b952f699 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
@@ -399,7 +399,7 @@ public class FilterEstimation extends
ExpressionVisitor<StatsDeriveResult, Estim
boolean isNotIn = context != null && context.isNot;
Expression compareExpr = inPredicate.getCompareExpr();
ColumnStatistic compareExprStats =
ExpressionEstimation.estimate(compareExpr, inputStats);
- if (compareExprStats == ColumnStatistic.UNKNOWN) {
+ if (compareExprStats == ColumnStatistic.DEFAULT) {
return inputStats;
}
List<Expression> options = inPredicate.getOptions();
@@ -427,7 +427,7 @@ public class FilterEstimation extends
ExpressionVisitor<StatsDeriveResult, Estim
if (isNotIn) {
for (Expression option : options) {
ColumnStatistic optionStats =
ExpressionEstimation.estimate(option, inputStats);
- if (optionStats == ColumnStatistic.UNKNOWN) {
+ if (optionStats == ColumnStatistic.DEFAULT) {
continue;
}
double validOptionNdv =
compareExprStats.ndvIntersection(optionStats);
@@ -441,7 +441,7 @@ public class FilterEstimation extends
ExpressionVisitor<StatsDeriveResult, Estim
} else {
for (Expression option : options) {
ColumnStatistic optionStats =
ExpressionEstimation.estimate(option, inputStats);
- if (optionStats == ColumnStatistic.UNKNOWN) {
+ if (optionStats == ColumnStatistic.DEFAULT) {
validInOptCount++;
continue;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index 4005979849..8f0a62ee5c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -295,7 +295,7 @@ public class StatsCalculator extends
DefaultPlanVisitor<StatsDeriveResult, Void>
}
ColumnStatistic statistic =
Env.getCurrentEnv().getStatisticsCache().getColumnStatistics(table.getId(),
colName);
- if (statistic == ColumnStatistic.UNKNOWN) {
+ if (statistic == ColumnStatistic.DEFAULT) {
if (card == -1) {
card = roughlyEstimatedCard(scan);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculatorV2.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculatorV2.java
index a3b97c01e3..4790143056 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculatorV2.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculatorV2.java
@@ -289,9 +289,6 @@ public class StatsCalculatorV2 extends
DefaultPlanVisitor<StatsDeriveResult, Voi
}
ColumnStatistic statistic =
Env.getCurrentEnv().getStatisticsCache().getColumnStatistics(table.getId(),
colName);
- if (statistic == ColumnStatistic.UNKNOWN) {
- statistic = ColumnStatistic.DEFAULT;
- }
rowCount = statistic.count;
columnStatisticMap.put(slotReference.getExprId(), statistic);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
index b91705bb5d..b5b331e4c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
@@ -38,12 +38,8 @@ public class ColumnStatistic {
private static final Logger LOG = LogManager.getLogger(StmtExecutor.class);
- public static ColumnStatistic UNKNOWN = new
ColumnStatisticBuilder().setCount(Double.NaN).setNdv(Double.NaN)
-
.setAvgSizeByte(Double.NaN).setNumNulls(Double.NaN).setDataSize(Double.NaN)
-
.setMinValue(Double.NaN).setMaxValue(Double.NaN).setMinExpr(null).setMaxExpr(null).build();
-
public static ColumnStatistic DEFAULT = new
ColumnStatisticBuilder().setAvgSizeByte(1).setNdv(1)
-
.setNumNulls(1).setCount(1).setMaxValue(Double.MAX_VALUE).setMinValue(Double.MIN_VALUE)
+
.setNumNulls(1).setCount(1).setMaxValue(Double.MAX_VALUE).setMinValue(Double.MIN_VALUE).setSelectivity(1.0)
.build();
public static final Set<Type> MAX_MIN_UNSUPPORTED_TYPE = new HashSet<>();
@@ -118,7 +114,7 @@ public class ColumnStatistic {
if (col == null) {
LOG.warn("Failed to deserialize column statistics,
column:{}.{}.{}.{} not exists",
catalogId, dbID, tblId, colName);
- return ColumnStatistic.UNKNOWN;
+ return ColumnStatistic.DEFAULT;
}
String min = resultRow.getColumnValue("min");
String max = resultRow.getColumnValue("max");
@@ -131,7 +127,7 @@ public class ColumnStatistic {
} catch (Exception e) {
e.printStackTrace();
LOG.warn("Failed to deserialize column statistics, column not
exists", e);
- return ColumnStatistic.UNKNOWN;
+ return ColumnStatistic.DEFAULT;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java
index c20098e55c..48dd0b2d52 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java
@@ -47,7 +47,7 @@ public class ColumnStatisticBuilder {
this.selectivity = columnStatistic.selectivity;
this.minExpr = columnStatistic.minExpr;
this.maxExpr = columnStatistic.maxExpr;
- this.buildOnUnknown = columnStatistic == ColumnStatistic.UNKNOWN;
+ this.buildOnUnknown = columnStatistic == ColumnStatistic.DEFAULT;
}
public ColumnStatisticBuilder setCount(double count) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
index 0b39cde495..1a97045481 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
@@ -39,7 +39,7 @@ public class StatisticsCache {
public ColumnStatistic getColumnStatistics(long tblId, String colName) {
if (ConnectContext.get().getSessionVariable().internalSession) {
- return ColumnStatistic.UNKNOWN;
+ return ColumnStatistic.DEFAULT;
}
StatisticsCacheKey k = new StatisticsCacheKey(tblId, colName);
CompletableFuture<ColumnStatistic> f = cache.get(k);
@@ -48,10 +48,10 @@ public class StatisticsCache {
return f.get();
} catch (Exception e) {
LOG.warn("Unexpected exception while returning
ColumnStatistic", e);
- return ColumnStatistic.UNKNOWN;
+ return ColumnStatistic.DEFAULT;
}
}
- return ColumnStatistic.UNKNOWN;
+ return ColumnStatistic.DEFAULT;
}
// TODO: finish this method.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
index 1e86b5a7ac..e5c85b4b33 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
@@ -81,7 +81,7 @@ public class StatisticsRepository {
public static ColumnStatistic queryColumnStatisticsByName(long tableId,
String colName) {
ResultRow resultRow = queryColumnStatisticById(tableId, colName);
if (resultRow == null) {
- return ColumnStatistic.UNKNOWN;
+ return ColumnStatistic.DEFAULT;
}
return ColumnStatistic.fromResultRow(resultRow);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java
index eb45e9c681..2d41c68599 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java
@@ -47,7 +47,7 @@ public class CacheTest extends TestWithFeService {
try {
Thread.sleep(50);
} catch (InterruptedException e) {
- return ColumnStatistic.UNKNOWN;
+ return ColumnStatistic.DEFAULT;
}
return ColumnStatistic.DEFAULT;
});
@@ -55,7 +55,7 @@ public class CacheTest extends TestWithFeService {
};
StatisticsCache statisticsCache = new StatisticsCache();
ColumnStatistic c = statisticsCache.getColumnStatistics(1, "col");
- Assertions.assertEquals(c, ColumnStatistic.UNKNOWN);
+ Assertions.assertEquals(c, ColumnStatistic.DEFAULT);
Thread.sleep(100);
c = statisticsCache.getColumnStatistics(1, "col");
Assertions.assertEquals(c, ColumnStatistic.DEFAULT);
@@ -118,7 +118,7 @@ public class CacheTest extends TestWithFeService {
};
StatisticsCache statisticsCache = new StatisticsCache();
ColumnStatistic columnStatistic =
statisticsCache.getColumnStatistics(0, "col");
- Assertions.assertEquals(ColumnStatistic.UNKNOWN, columnStatistic);
+ Assertions.assertEquals(ColumnStatistic.DEFAULT, columnStatistic);
Thread.sleep(100);
columnStatistic = statisticsCache.getColumnStatistics(0, "col");
Assertions.assertEquals(1, columnStatistic.count);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
