englefly commented on code in PR #67549:
URL: https://github.com/apache/doris/pull/67549#discussion_r4238200321
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboStatsCalculator.java:
##########
@@ -85,17 +233,201 @@ public Statistics computeAggregate(Aggregate<? extends
Plan> aggregate, Statisti
// e.g, logical one likes "count(*) AS `count(*)`#4"
// local physical one likes "partial_count(*) AS
`partial_count(*)`#5"
// global physical one likes "count(partial_count(*)#5) AS
`count(*)`#4"
- return getStatsFromHboPlanStats((AbstractPlan) aggregate, legacyStats);
+ return getStatsFromHboPlanStats((AbstractPlan) aggregate, legacyStats,
+ GroupStructInfo.LiteralMode.NO_LITERAL, null);
+ }
+
+ /**
+ * Apply a pinned entry for the given plan node: exact filter form, then
constant agnostic form
+ * (join / aggregation only have the latter). Returns null when no entry
applies.
+ *
+ * <p>The fingerprint of an entry does not contain the data state of its
tables any more, so an
+ * entry keeps matching while a table grows; whether it may be used is
decided here, by comparing
+ * the baseline the entry was recorded with against the data state of this
query
+ * (see {@link HboStructFreshness}): an entry whose data moved by more than
+ * {@code Config.hbo_row_count_change_ratio} is skipped, so the query
falls back to the optimizer
+ * estimation instead of applying a row count which no longer describes
the data.
+ *
+ * @param guardInputStats filter input statistics, required to evaluate
the FILTER_SMALL guard
+ * of a filter node (null for join / aggregation)
+ */
+ private Statistics applyPinnedStats(AbstractPlan planNode, Statistics
delegateStats,
+ Statistics guardInputStats, GroupStructInfo.LiteralMode[]
lookupModes) {
+ // the fingerprint of the node is what an entry is keyed by, and
building it walks the whole
+ // memo subtree: with an empty pinned cache every lookup is a
guaranteed miss, so the
+ // fingerprint is not built at all (this is the common case in
production, where hbo
+ // optimization is enabled but nothing was injected)
+ if
(!Env.getCurrentEnv().getHboPlanStatisticsManager().hasAnyPinnedStatistics()) {
+ return null;
+ }
+ // the relation key of the group is a necessary condition for a match
and costs one step per
+ // group, while the fingerprint walks the whole memo sub tree: when no
entry was recorded for
+ // a sub tree which reads exactly these tables, no fingerprint of this
node can match
+ String relationKey = GroupStructInfo.relationKeyOfPlanNode(planNode,
null);
+ if (relationKey != null &&
!Env.getCurrentEnv().getHboPlanStatisticsManager()
+ .mayHavePinnedEntryForRelations(relationKey)) {
+ return null;
+ }
+ for (GroupStructInfo.LiteralMode mode : lookupModes) {
+ Optional<GroupStructInfo> structInfoOpt =
GroupStructInfo.structInfoOfPlanNode(planNode, null, mode);
+ if (!structInfoOpt.isPresent()) {
+ continue;
+ }
+ GroupStructInfo structInfo = structInfoOpt.get();
+ String fingerprint = structInfo.getFingerprint();
+ Optional<HboPlanStatisticsManager.PinnedHboStatistics> pinnedOpt =
Env.getCurrentEnv()
+
.getHboPlanStatisticsManager().getPinnedPlanStatistics(fingerprint);
+ if (!pinnedOpt.isPresent()) {
+ continue;
+ }
+ HboPlanStatisticsManager.PinnedHboStatistics pinned =
pinnedOpt.get();
+ // report which literal mode of the injected entry matched (and,
for FILTER_SMALL, why it
+ // was skipped) through the explain annotation; its type is read
from the entry itself
+ recordPinnedLiteralMode(fingerprint,
+ mode == GroupStructInfo.LiteralMode.NO_LITERAL ?
"no_literal" : "with_literal");
+ if (pinned.getType() ==
HboPlanStatisticsManager.PinnedType.FILTER_SMALL
+ && guardInputStats != null
+ &&
!isExtremeSmallFilterEstimate(delegateStats.getRowCount(),
+ guardInputStats.getRowCount())) {
+ // the estimate is not in the pathological regime this entry
was injected for:
+ // skip it (and try the coarser granularity / learned matching
instead)
+ recordGuardSkip(fingerprint, delegateStats.getRowCount(),
guardInputStats.getRowCount());
+ continue;
+ }
+ // the entry is judged against a data state taken now, not against
the one the memo cached
+ // when this group was looked up the first time: a load which
committed while the query is
+ // being optimized would otherwise be invisible. The cached struct
info stays the source of
+ // the fingerprint, which does not depend on any data state. An
entry which records no data
+ // state at all is applied as it always was, so there is nothing
to walk for.
+ HboStructFreshness freshness;
+ if
(HboStructFreshness.hasRecordedDataState(pinned.getStructCanonical())) {
+ Optional<GroupStructInfo> dataState =
GroupStructInfo.dataStateOfPlanNode(planNode, null, mode);
+ freshness =
HboStructFreshness.between(pinned.getStructCanonical(),
+ dataState.isPresent() ? dataState.get() : structInfo);
+ } else {
+ freshness =
HboStructFreshness.between(pinned.getStructCanonical(), structInfo.getScans());
+ }
+ if (freshness.isStale()) {
+ // the data this entry was measured on moved too far (or its
state cannot be
+ // verified any more): do not apply the recorded row count,
keep looking
+ recordPinnedSkip(fingerprint, freshness.getSummary());
+ continue;
+ }
+ recordPinnedApplyState(fingerprint, freshness.getSummary());
+ pinned.recordUse(freshness);
+ return delegateStats.withRowCountAndHboFlag(pinned.getRows());
+ }
+ return null;
}
- private Statistics getStatsFromHboPlanStats(AbstractPlan planNode,
Statistics delegateStats) {
- PlanNodeAndHash planNodeAndHash = HboUtils.getPlanNodeHash(planNode);
- RecentRunsPlanStatistics planStatistics =
hboPlanStatisticsProvider.getHboPlanStats(planNodeAndHash);
- PlanStatistics matchedPlanStatistics =
HboUtils.getMatchedPlanStatistics(planStatistics,
+ private Statistics getStatsFromHboPlanStats(AbstractPlan planNode,
Statistics delegateStats,
+ GroupStructInfo.LiteralMode mode, Statistics guardInputStats) {
+ Statistics pinnedStats = applyPinnedStats(planNode, delegateStats,
guardInputStats,
+ JOIN_AGG_LOOKUP_MODES);
+ if (pinnedStats != null) {
+ return pinnedStats;
+ }
+ Statistics learnedStats = applyLearnedStats(planNode, mode,
delegateStats);
+ return learnedStats == null ? delegateStats : learnedStats;
+ }
+
+ private Statistics applyLearnedStats(AbstractPlan planNode,
GroupStructInfo.LiteralMode mode,
+ Statistics delegateStats) {
+ // same reasoning as in applyPinnedStats: the learned key contains the
hbo fingerprint of the
+ // node, which is only worth building when the learned cache holds
something at all
+ if (!hboPlanStatisticsProvider.hasAnyHboPlanStats()) {
+ return null;
+ }
+ Optional<PlanNodeAndHash> planNodeAndHashOpt =
HboUtils.getHboPlanNodeAndHash(planNode, mode);
+ if (!planNodeAndHashOpt.isPresent() ||
!planNodeAndHashOpt.get().getHash().isPresent()) {
+ return null;
+ }
+ RecentRunsPlanStatistics planStatistics =
hboPlanStatisticsProvider.getHboPlanStats(planNodeAndHashOpt.get());
+ RecentRunsPlanStatisticsEntry matchedEntry =
HboUtils.getMatchedEntry(planStatistics,
cascadesContext.getConnectContext());
- if (matchedPlanStatistics != null) {
- delegateStats =
delegateStats.withRowCountAndHboFlag(matchedPlanStatistics.getOutputRows());
+ if (matchedEntry == null) {
+ return null;
+ }
+ // a learned key does not contain the data state of its tables either,
so the entry is judged
+ // by the rows it recorded for its input tables: an entry published
before a large data change
+ // must not keep its old row count (see
HboStructFreshness.ofLearnedEntry).
+ //
+ // Only a scan node reports the rows it read before its own
predicates, which is the number a
+ // catalog row count can be compared with; every other node reports
what its input produced
+ // after those predicates, where a ratio against the rows of the data
would be off by the
+ // selectivity of that node. Guarding those would reject almost every
selective entry, so they
+ // are left unguarded (the publish path would have to record the rows
of the node's input
+ // tables for that).
+ String hash = planNodeAndHashOpt.get().getHash().get();
+ if (planNode instanceof OlapScan) {
Review Comment:
认可，这正是我在分支里记为 follow-up 的一条：新的 group 指纹不含 version，而 learned 守卫目前只对 OlapScan
生效，join/agg 的 learned 条目在输入暴涨后仍会用旧输出行数。正确修法需要发布路径为每个输入表记录
inputRows、读侧对任意节点做同样比较，属独立改动，我没有塞进本 PR。
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboStatsCalculator.java:
##########
@@ -85,17 +233,201 @@ public Statistics computeAggregate(Aggregate<? extends
Plan> aggregate, Statisti
// e.g, logical one likes "count(*) AS `count(*)`#4"
// local physical one likes "partial_count(*) AS
`partial_count(*)`#5"
// global physical one likes "count(partial_count(*)#5) AS
`count(*)`#4"
- return getStatsFromHboPlanStats((AbstractPlan) aggregate, legacyStats);
+ return getStatsFromHboPlanStats((AbstractPlan) aggregate, legacyStats,
+ GroupStructInfo.LiteralMode.NO_LITERAL, null);
+ }
+
+ /**
+ * Apply a pinned entry for the given plan node: exact filter form, then
constant agnostic form
+ * (join / aggregation only have the latter). Returns null when no entry
applies.
+ *
+ * <p>The fingerprint of an entry does not contain the data state of its
tables any more, so an
+ * entry keeps matching while a table grows; whether it may be used is
decided here, by comparing
+ * the baseline the entry was recorded with against the data state of this
query
+ * (see {@link HboStructFreshness}): an entry whose data moved by more than
+ * {@code Config.hbo_row_count_change_ratio} is skipped, so the query
falls back to the optimizer
+ * estimation instead of applying a row count which no longer describes
the data.
+ *
+ * @param guardInputStats filter input statistics, required to evaluate
the FILTER_SMALL guard
+ * of a filter node (null for join / aggregation)
+ */
+ private Statistics applyPinnedStats(AbstractPlan planNode, Statistics
delegateStats,
+ Statistics guardInputStats, GroupStructInfo.LiteralMode[]
lookupModes) {
+ // the fingerprint of the node is what an entry is keyed by, and
building it walks the whole
+ // memo subtree: with an empty pinned cache every lookup is a
guaranteed miss, so the
+ // fingerprint is not built at all (this is the common case in
production, where hbo
+ // optimization is enabled but nothing was injected)
+ if
(!Env.getCurrentEnv().getHboPlanStatisticsManager().hasAnyPinnedStatistics()) {
+ return null;
+ }
+ // the relation key of the group is a necessary condition for a match
and costs one step per
+ // group, while the fingerprint walks the whole memo sub tree: when no
entry was recorded for
+ // a sub tree which reads exactly these tables, no fingerprint of this
node can match
+ String relationKey = GroupStructInfo.relationKeyOfPlanNode(planNode,
null);
+ if (relationKey != null &&
!Env.getCurrentEnv().getHboPlanStatisticsManager()
+ .mayHavePinnedEntryForRelations(relationKey)) {
+ return null;
+ }
+ for (GroupStructInfo.LiteralMode mode : lookupModes) {
+ Optional<GroupStructInfo> structInfoOpt =
GroupStructInfo.structInfoOfPlanNode(planNode, null, mode);
+ if (!structInfoOpt.isPresent()) {
+ continue;
+ }
+ GroupStructInfo structInfo = structInfoOpt.get();
+ String fingerprint = structInfo.getFingerprint();
+ Optional<HboPlanStatisticsManager.PinnedHboStatistics> pinnedOpt =
Env.getCurrentEnv()
+
.getHboPlanStatisticsManager().getPinnedPlanStatistics(fingerprint);
+ if (!pinnedOpt.isPresent()) {
+ continue;
+ }
+ HboPlanStatisticsManager.PinnedHboStatistics pinned =
pinnedOpt.get();
+ // report which literal mode of the injected entry matched (and,
for FILTER_SMALL, why it
+ // was skipped) through the explain annotation; its type is read
from the entry itself
+ recordPinnedLiteralMode(fingerprint,
+ mode == GroupStructInfo.LiteralMode.NO_LITERAL ?
"no_literal" : "with_literal");
+ if (pinned.getType() ==
HboPlanStatisticsManager.PinnedType.FILTER_SMALL
+ && guardInputStats != null
+ &&
!isExtremeSmallFilterEstimate(delegateStats.getRowCount(),
+ guardInputStats.getRowCount())) {
+ // the estimate is not in the pathological regime this entry
was injected for:
+ // skip it (and try the coarser granularity / learned matching
instead)
+ recordGuardSkip(fingerprint, delegateStats.getRowCount(),
guardInputStats.getRowCount());
+ continue;
+ }
+ // the entry is judged against a data state taken now, not against
the one the memo cached
+ // when this group was looked up the first time: a load which
committed while the query is
+ // being optimized would otherwise be invisible. The cached struct
info stays the source of
+ // the fingerprint, which does not depend on any data state. An
entry which records no data
+ // state at all is applied as it always was, so there is nothing
to walk for.
+ HboStructFreshness freshness;
+ if
(HboStructFreshness.hasRecordedDataState(pinned.getStructCanonical())) {
+ Optional<GroupStructInfo> dataState =
GroupStructInfo.dataStateOfPlanNode(planNode, null, mode);
+ freshness =
HboStructFreshness.between(pinned.getStructCanonical(),
+ dataState.isPresent() ? dataState.get() : structInfo);
Review Comment:
已在本轮修复：live data state 读不到时不再回退到缓存的（可能很旧的）baseline 判 LIVE，而是按“无可比较”处理为
unknown。注意我保留了 unknown → 仍然应用（soft
strategy）：一次临时元数据失败不该让用户注入的条目失效；这条的实质（不能用旧基线判 LIVE）已解决。
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboStructFreshness.java:
##########
@@ -0,0 +1,579 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.stats;
+
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.Config;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
+import org.apache.doris.rpc.RpcException;
+import org.apache.doris.statistics.hbo.PlanStatistics;
+import org.apache.doris.statistics.hbo.ScanPlanStatistics;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Whether a hbo entry may still be applied: the recorded data state of an
entry (the baseline of
+ * every scan of its struct info, see {@link HboScanDescriptor}) compared with
the data state of the
+ * query which wants to use it.
+ *
+ * <p>A hbo fingerprint does not contain any data state any more, so an entry
keeps matching while a
+ * table grows. The read side therefore has to decide whether the injected row
count still describes
+ * the data it was measured on:
+ * <ul>
+ * <li>{@link #STATE_LIVE}: the data the entry was measured on is still the
current one, the entry
+ * is applied;</li>
+ * <li>{@link #STATE_DRIFTED}: the data moved, but by at most
+ * {@code Config.hbo_row_count_change_ratio} of the recorded row count
(10% by default): the
+ * entry is applied, which is the point of the tolerance - a table which
merely keeps growing
+ * (e.g. a few new rows in a partition) does not invalidate a measured
row count;</li>
+ * <li>{@link #STATE_STALE}: the data moved further than that: the entry is
<b>not</b> applied, and
+ * the query falls back to the optimizer estimation;</li>
+ * <li>{@link #STATE_UNKNOWN}: nothing can be compared (the entry records no
row count, or the
+ * current one cannot be read / was obtained a different way), so there
is nothing to judge: the
+ * entry is applied as it always was.</li>
+ * </ul>
+ * The recorded row count is never scaled: an entry either applies with the
value the user measured
+ * or does not apply at all ("soft strategy" - a user who needs the exact
recorded state in any case
+ * should not inject hbo statistics for that query). Setting
+ * {@code hbo_row_count_change_ratio} to 0 or less switches the verdict back
to the strict semantics
+ * hbo had before the baseline existed: the recorded visible version has to
still be the current one.
+ *
+ * <p>The rows are compared <b>before</b> the version, and only against a
number which was obtained
+ * the same way ({@link HboScanDescriptor.RowsKind}): the visible version is a
property of the table,
+ * so two different partition selections of one table share it while reading
completely different
+ * amounts of data, and a measured row count must not be compared with one
that was derived from the
+ * table average. Whenever nothing can be compared - a hand written struct
literal, an entry whose
+ * rows were never measured, a current row count which cannot be read, or two
numbers of different
+ * kinds - the state is {@link #STATE_UNKNOWN} and the entry is applied: a
transient failure to read
+ * the catalog must not drop an entry the user injected. The version is only
decisive in strict mode
+ * ({@code hbo_row_count_change_ratio <= 0}).
+ *
+ * <p>A scan is judged by the rows of its <b>selected</b> partitions, so
pruning decides which data
+ * the entry depends on: an unrelated partition which grows does not
invalidate a pruned entry.
+ * {@code HBO SHOW STATISTICS} and {@code HBO DELETE STALE STATISTICS} cannot
run the query of an
+ * entry, so there the recorded state is compared with the table as a whole;
an entry whose struct
+ * prunes partitions can therefore only be judged by rows when it was recorded
as a whole table scan
+ * (its state is reported as unknown otherwise, and such an entry is kept
unless the user asks for
+ * {@code OLDER_THAN}).
+ *
+ * <p>Learned (profile collected) entries carry the input table statistics of
the run they were
+ * measured in, so they are judged by the same tolerance over those recorded
rows; an entry injected
+ * by {@code HBO SET LEARNED STATISTICS} carries no such statistics and is
applied as it always was.
+ */
+public class HboStructFreshness {
+ /** The recorded data state is still the current one. */
+ public static final String STATE_LIVE = "live";
+ /** The data moved, but within the tolerance of {@code
Config.hbo_row_count_change_ratio}. */
+ public static final String STATE_DRIFTED = "drifted";
+ /** The data moved further than the tolerance: the entry must not be
applied. */
+ public static final String STATE_STALE = "stale";
+ /** The entry records no data state (or it cannot be read any more):
nothing to judge. */
+ public static final String STATE_UNKNOWN = "unknown";
+
+ private static final Logger LOG =
LogManager.getLogger(HboStructFreshness.class);
+
+ private final String state;
+ // the recorded baseline and the current state, both rendered for display
+ private final String recorded;
+ private final String live;
+ // the row counts of the scan which decided the state, {@link
HboScanDescriptor#UNKNOWN} when the
+ // state was decided by something else (e.g. a version which is not
readable)
+ private final long recordedRows;
+ private final long liveRows;
+
+ private HboStructFreshness(String state, String recorded, String live,
long recordedRows, long liveRows) {
+ this.state = state;
+ this.recorded = recorded;
+ this.live = live;
+ this.recordedRows = recordedRows;
+ this.liveRows = liveRows;
+ }
+
+ public String getState() {
+ return state;
+ }
+
+ public boolean isStale() {
+ return STATE_STALE.equals(state);
+ }
+
+ public boolean isUnknown() {
+ return STATE_UNKNOWN.equals(state);
+ }
+
+ public boolean isDrifted() {
+ return STATE_DRIFTED.equals(state);
+ }
+
+ /** The data state the entry was recorded in, e.g. {@code
hbo_test.t1:v3,r1000,p1/5}. */
+ public String getRecorded() {
+ return recorded;
+ }
+
+ /** The data state the entry is compared with (empty when it cannot be
read). */
+ public String getLive() {
+ return live;
+ }
+
+ /**
+ * The verdict as one token, printed after {@code used=} / {@code
skipped=} in the explain
+ * annotation, e.g. {@code drifted(rows=1030,rec=1000,+3.0%)}.
+ */
+ public String getSummary() {
+ if (!STATE_DRIFTED.equals(state) && !STATE_STALE.equals(state)) {
+ return state;
+ }
+ if (recordedRows == HboScanDescriptor.UNKNOWN || liveRows ==
HboScanDescriptor.UNKNOWN) {
+ return state;
+ }
+ return state + "(rows=" + liveRows + ",rec=" + recordedRows + ","
+ + changeText(recordedRows, liveRows) + ")";
+ }
+
+ /** The {@code now} part of {@code HBO SHOW STATISTICS}: the state read
from the catalog. */
+ public String getLiveDetail() {
+ if (live.isEmpty()) {
+ return "-";
+ }
+ if (recordedRows == HboScanDescriptor.UNKNOWN || liveRows ==
HboScanDescriptor.UNKNOWN) {
+ return live;
+ }
+ return live + "," + changeText(recordedRows, liveRows);
+ }
+
+ /**
+ * Whether the entry records any data state at all. An entry which does
not (a struct literal
+ * typed by hand, or one copied from a plan whose tables were never
reported) is always applied,
+ * so a caller does not have to compute the state of the current query to
find that out.
+ */
+ public static boolean hasRecordedDataState(String recordedCanonical) {
+ for (HboScanDescriptor scan :
HboScanDescriptor.parseAll(recordedCanonical)) {
+ if (scan.hasVisibleVersion() || scan.hasScanRows()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Judge an entry against the current query: the data state recorded in
{@code recordedCanonical}
+ * (the struct info of the entry) against the state of {@code
liveCanonical} (the struct info the
+ * read side just computed for the group). This is the decision of the
read side.
+ */
+ public static HboStructFreshness between(String recordedCanonical,
GroupStructInfo liveStructInfo) {
+ return between(recordedCanonical, liveStructInfo.getScans());
+ }
+
+ /** The same comparison for a caller which has the scans of the current
query only. */
+ static HboStructFreshness between(String recordedCanonical,
List<HboScanDescriptor> liveScans) {
+ List<HboScanDescriptor> recordedScans =
HboScanDescriptor.parseAll(recordedCanonical);
+ String liveText = renderScans(liveScans);
+ if (recordedScans.isEmpty()) {
+ // an entry without a baseline (a hand written struct literal):
nothing to judge
+ return new HboStructFreshness(STATE_UNKNOWN, "", liveText,
+ HboScanDescriptor.UNKNOWN, HboScanDescriptor.UNKNOWN);
+ }
+ if (recordedScans.size() != liveScans.size()) {
+ // the entry describes a different number of scans than the group:
cannot be compared
+ return new HboStructFreshness(STATE_UNKNOWN,
renderScans(recordedScans), liveText,
+ HboScanDescriptor.UNKNOWN, HboScanDescriptor.UNKNOWN);
+ }
+ String state = STATE_LIVE;
+ String recordedText = renderScans(recordedScans);
+
+ long recordedRows = HboScanDescriptor.UNKNOWN;
+ long liveRows = HboScanDescriptor.UNKNOWN;
+ for (int i = 0; i < recordedScans.size(); i++) {
+ HboScanDescriptor recorded = recordedScans.get(i);
+ HboScanDescriptor live = liveScans.get(i);
+ String scanState;
+ if (!recorded.getTable().equals(live.getTable())) {
+ // the same fingerprint describes the same scans in the same
order, so this cannot
+ // happen for an entry of this group: refuse to judge rather
than guess
+ scanState = STATE_UNKNOWN;
+ } else {
+ scanState = stateOf(recorded, live);
+ }
+ if (worse(scanState, state).equals(scanState)) {
+ state = scanState;
+ recordedRows = recorded.getScanRows();
+ liveRows = live.getScanRows();
+ }
+ }
+ return new HboStructFreshness(state, recordedText, liveText,
recordedRows, liveRows);
+ }
+
+ /**
+ * Judge an entry against the catalog as a whole (used by {@code HBO SHOW
STATISTICS} and
+ * {@code HBO DELETE STALE STATISTICS}, which cannot run the query of an
entry): the recorded
+ * visible version and row count of every table against the current ones.
+ *
+ * <p>Row counts are only comparable when the scan read the whole table
(no partition was pruned
+ * away), because only then the recorded rows are the rows of the table.
+ */
+ public static HboStructFreshness of(String canonicalStructInfo) {
+ List<HboScanDescriptor> recordedScans =
HboScanDescriptor.parseAll(canonicalStructInfo);
+ if (recordedScans.isEmpty()) {
+ return new HboStructFreshness(STATE_UNKNOWN, "", "",
HboScanDescriptor.UNKNOWN,
+ HboScanDescriptor.UNKNOWN);
+ }
+ String state = STATE_LIVE;
+ // the current state of every table, de duplicated like the recorded
side (a self join reads
+ // the same table twice, which would only repeat the same numbers)
+ Set<String> liveTexts = new LinkedHashSet<>();
+ long recordedRows = HboScanDescriptor.UNKNOWN;
+ long liveRows = HboScanDescriptor.UNKNOWN;
+ for (HboScanDescriptor recorded : recordedScans) {
+ OlapTable table = resolveTable(recorded.getTable());
+ if (table == null) {
+ if (worse(STATE_UNKNOWN, state).equals(STATE_UNKNOWN)) {
+ state = STATE_UNKNOWN;
+ }
+ liveTexts.add(recorded.getTable() + ":-");
+ continue;
+ }
+ if (table.getIndexIdList().size() > 1 &&
!recorded.isPartitionSelectionComplete()) {
+ // the table has more than one index (a rollup) and the entry
records only a partition
+ // count, so this statement cannot tell which index its row
count came from: judging it
+ // against the base index could be wrong in both directions,
so it is left to a query
+ liveTexts.add(recorded.getTable() + ":-");
+ if (worse(STATE_UNKNOWN, state).equals(STATE_UNKNOWN)) {
+ state = STATE_UNKNOWN;
+ }
+ continue;
+ }
+ long currentVersion = visibleVersionOf(table);
+ // the rows of the whole table, computed the same way the recorded
rows of a whole table
+ // scan were computed, so the two numbers describe the same thing
+ HboScanDescriptor.Rows liveRowsOfTable =
HboScanDescriptor.rowsOf(table,
+ new ArrayList<>(table.getPartitionIds()),
table.getBaseIndexId());
+ long currentRows = liveRowsOfTable.getRows();
+ HboScanDescriptor.RowsKind currentRowsKind =
liveRowsOfTable.getKind();
+ liveTexts.add(recorded.getTable() + ":v" + currentVersion
+ + (currentRows == HboScanDescriptor.UNKNOWN ? ""
+ : (currentRowsKind ==
HboScanDescriptor.RowsKind.MEASURED ? ",r" : ",e")
+ + currentRows));
+ String scanState;
+ if (!recorded.hasVisibleVersion() && !recorded.hasScanRows()) {
+ // nothing was recorded for this scan: it applies as it always
was, nothing to judge
+ scanState = STATE_UNKNOWN;
+ currentRows = HboScanDescriptor.UNKNOWN;
+ } else if (threshold() <= 0) {
+ scanState = recorded.hasVisibleVersion() && currentVersion ==
recorded.getVisibleVersion()
Review Comment:
已在本轮修复：严格模式下当前 visible version 读不到（-1）时不再当成版本不一致判 STALE，改为 unknown，避免一次临时云上
RPC 失败导致 DELETE STALE 误删条目。
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
