englefly commented on code in PR #69322: URL: https://github.com/apache/doris/pull/69322#discussion_r4361412608
########## fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboStructFreshness.java: ########## @@ -0,0 +1,682 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.Config; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.rpc.RpcException; +import org.apache.doris.statistics.hbo.PlanStatistics; +import org.apache.doris.statistics.hbo.ScanPlanStatistics; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Whether a hbo entry may still be applied: the recorded data state of an entry (the baseline of + * every scan of its struct info, see {@link HboScanDescriptor}) compared with the data state of the + * query which wants to use it. + * + * <p>A hbo fingerprint does not contain any data state any more, so an entry keeps matching while a + * table grows. The read side therefore has to decide whether the injected row count still describes + * the data it was measured on: + * <ul> + * <li>{@link #STATE_LIVE}: the data the entry was measured on is still the current one, the entry + * is applied;</li> + * <li>{@link #STATE_DRIFTED}: the data moved, but by at most + * {@code Config.hbo_row_count_change_ratio} of the recorded row count (10% by default): the + * entry is applied, which is the point of the tolerance - a table which merely keeps growing + * (e.g. a few new rows in a partition) does not invalidate a measured row count;</li> + * <li>{@link #STATE_STALE}: the data moved further than that: the entry is <b>not</b> applied, and + * the query falls back to the optimizer estimation;</li> + * <li>{@link #STATE_UNKNOWN}: nothing can be compared (the entry records no row count, or the + * current one cannot be read / was obtained a different way), so there is nothing to judge: the + * entry is applied as it always was.</li> + * </ul> + * The recorded row count is never scaled: an entry either applies with the value the user measured + * or does not apply at all ("soft strategy" - a user who needs the exact recorded state in any case + * should not inject hbo statistics for that query). Setting + * {@code hbo_row_count_change_ratio} to 0 or less switches the verdict back to the strict semantics + * hbo had before the baseline existed: the recorded visible version has to still be the current one. + * + * <p>The rows are compared <b>before</b> the version, and only against a number which was obtained + * the same way ({@link HboScanDescriptor.RowsKind}): the visible version is a property of the table, + * so two different partition selections of one table share it while reading completely different + * amounts of data, and a measured row count must not be compared with one that was derived from the + * table average. Whenever nothing can be compared - a hand written struct literal, an entry whose + * rows were never measured, a current row count which cannot be read, or two numbers of different + * kinds - the state is {@link #STATE_UNKNOWN} and the entry is applied: a transient failure to read + * the catalog must not drop an entry the user injected. The version is only decisive in strict mode + * ({@code hbo_row_count_change_ratio <= 0}). + * + * <p>A scan is judged by the rows of its <b>selected</b> partitions, so pruning decides which data + * the entry depends on: an unrelated partition which grows does not invalidate a pruned entry. + * {@code HBO SHOW STATISTICS} and {@code HBO DELETE STALE STATISTICS} cannot run the query of an + * entry, so there the recorded state is compared with the table as a whole; an entry whose struct + * prunes partitions can therefore only be judged by rows when it was recorded as a whole table scan + * (its state is reported as unknown otherwise, and such an entry is kept unless the user asks for + * {@code OLDER_THAN}). + * + * <p>Learned (profile collected) entries carry the input table statistics of the run they were + * measured in, so they are judged by the same tolerance over those recorded rows; an entry injected + * by {@code HBO SET LEARNED STATISTICS} carries no such statistics and is applied as it always was. + */ +public class HboStructFreshness { + /** The recorded data state is still the current one. */ + public static final String STATE_LIVE = "live"; + /** The data moved, but within the tolerance of {@code Config.hbo_row_count_change_ratio}. */ + public static final String STATE_DRIFTED = "drifted"; + /** The data moved further than the tolerance: the entry must not be applied. */ + public static final String STATE_STALE = "stale"; + /** The entry records no data state (or it cannot be read any more): nothing to judge. */ + public static final String STATE_UNKNOWN = "unknown"; + + private static final Logger LOG = LogManager.getLogger(HboStructFreshness.class); + + private final String state; + // the recorded baseline and the current state, both rendered for display + private final String recorded; + private final String live; + // the row counts of the scan which decided the state, {@link HboScanDescriptor#UNKNOWN} when the + // state was decided by something else (e.g. a version which is not readable) + private final long recordedRows; + private final long liveRows; + + private HboStructFreshness(String state, String recorded, String live, long recordedRows, long liveRows) { + this.state = state; + this.recorded = recorded; + this.live = live; + this.recordedRows = recordedRows; + this.liveRows = liveRows; + } + + public String getState() { + return state; + } + + public boolean isStale() { + return STATE_STALE.equals(state); + } + + public boolean isUnknown() { + return STATE_UNKNOWN.equals(state); + } + + public boolean isDrifted() { + return STATE_DRIFTED.equals(state); + } + + /** The data state the entry was recorded in, e.g. {@code hbo_test.t1:v3,r1000,p1/5}. */ + public String getRecorded() { + return recorded; + } + + /** The data state the entry is compared with (empty when it cannot be read). */ + public String getLive() { + return live; + } + + /** + * The verdict as one token, printed after {@code used=} / {@code skipped=} in the explain + * annotation, e.g. {@code drifted(rows=1030,rec=1000,+3.0%)}. + */ + public String getSummary() { + if (!STATE_DRIFTED.equals(state) && !STATE_STALE.equals(state)) { + return state; + } + if (recordedRows == HboScanDescriptor.UNKNOWN || liveRows == HboScanDescriptor.UNKNOWN) { + return state; + } + return state + "(rows=" + liveRows + ",rec=" + recordedRows + "," + + changeText(recordedRows, liveRows) + ")"; + } + + /** The {@code now} part of {@code HBO SHOW STATISTICS}: the state read from the catalog. */ + public String getLiveDetail() { + if (live.isEmpty()) { + return "-"; + } + if (recordedRows == HboScanDescriptor.UNKNOWN || liveRows == HboScanDescriptor.UNKNOWN) { + return live; + } + return live + "," + changeText(recordedRows, liveRows); + } + + /** + * Whether the entry records any data state at all. An entry which does not (a struct literal + * typed by hand, or one copied from a plan whose tables were never reported) is always applied, + * so a caller does not have to compute the state of the current query to find that out. + */ + public static boolean hasRecordedDataState(String recordedCanonical) { + for (HboScanDescriptor scan : HboScanDescriptor.parseAll(recordedCanonical)) { + if (scan.hasVisibleVersion() || scan.hasScanRows()) { + return true; + } + } + return false; + } + + /** + * Judge an entry against the current query: the data state recorded in {@code recordedCanonical} + * (the struct info of the entry) against the state of {@code liveCanonical} (the struct info the + * read side just computed for the group). This is the decision of the read side. + */ + public static HboStructFreshness between(String recordedCanonical, GroupStructInfo liveStructInfo) { + return between(recordedCanonical, liveStructInfo.getScans()); + } + + /** The same comparison for a caller which has the scans of the current query only. */ + static HboStructFreshness between(String recordedCanonical, List<HboScanDescriptor> liveScans) { + List<HboScanDescriptor> recordedScans = HboScanDescriptor.parseAll(recordedCanonical); + String liveText = renderScans(liveScans); + if (recordedScans.isEmpty()) { + // an entry without a baseline (a hand written struct literal): nothing to judge + return new HboStructFreshness(STATE_UNKNOWN, "", liveText, + HboScanDescriptor.UNKNOWN, HboScanDescriptor.UNKNOWN); + } + if (recordedScans.size() != liveScans.size()) { + // the entry describes a different number of scans than the group: cannot be compared + return new HboStructFreshness(STATE_UNKNOWN, renderScans(recordedScans), liveText, + HboScanDescriptor.UNKNOWN, HboScanDescriptor.UNKNOWN); + } + String state = STATE_LIVE; + String recordedText = renderScans(recordedScans); + + long recordedRows = HboScanDescriptor.UNKNOWN; + long liveRows = HboScanDescriptor.UNKNOWN; + for (int i = 0; i < recordedScans.size(); i++) { + HboScanDescriptor recorded = recordedScans.get(i); + HboScanDescriptor live = liveScans.get(i); + String scanState; + if (!recorded.getTable().equals(live.getTable())) { + // the same fingerprint describes the same scans in the same order, so this cannot + // happen for an entry of this group: refuse to judge rather than guess + scanState = STATE_UNKNOWN; + } else { + scanState = stateOf(recorded, live); + } + if (worse(scanState, state).equals(scanState)) { + state = scanState; + recordedRows = recorded.getScanRows(); + liveRows = live.getScanRows(); + } + } + return new HboStructFreshness(state, recordedText, liveText, recordedRows, liveRows); + } + + /** + * Judge an entry against the catalog as a whole (used by {@code HBO SHOW STATISTICS} and + * {@code HBO DELETE STALE STATISTICS}, which cannot run the query of an entry): the recorded + * visible version and row count of every table against the current ones. + * + * <p>Row counts are only comparable when the scan read the whole table (no partition was pruned + * away), because only then the recorded rows are the rows of the table. + */ + public static HboStructFreshness of(String canonicalStructInfo) { + List<HboScanDescriptor> recordedScans = HboScanDescriptor.parseAll(canonicalStructInfo); + if (recordedScans.isEmpty()) { + return new HboStructFreshness(STATE_UNKNOWN, "", "", HboScanDescriptor.UNKNOWN, + HboScanDescriptor.UNKNOWN); + } + String state = STATE_LIVE; + // the current state of every table, de duplicated like the recorded side (a self join reads + // the same table twice, which would only repeat the same numbers) + Set<String> liveTexts = new LinkedHashSet<>(); + long recordedRows = HboScanDescriptor.UNKNOWN; + long liveRows = HboScanDescriptor.UNKNOWN; + for (HboScanDescriptor recorded : recordedScans) { + OlapTable table = resolveTable(recorded.getTable()); + if (table != null && tableReplaced(recorded, table)) { + // the name now resolves to another table (the recorded one was dropped and created again): + // the entry describes data which is gone, and no plan of the new table can produce its + // key, so judging that key by the replacement's state would keep a dead row as LIVE + liveTexts.add(recorded.getTable() + ":replaced#" + table.getId()); + if (worse(STATE_STALE, state).equals(STATE_STALE)) { + state = STATE_STALE; + recordedRows = HboScanDescriptor.UNKNOWN; + liveRows = HboScanDescriptor.UNKNOWN; + } + continue; + } + if (table == null) { + if (worse(STATE_UNKNOWN, state).equals(STATE_UNKNOWN)) { + state = STATE_UNKNOWN; + } + liveTexts.add(recorded.getTable() + ":-"); + continue; + } + if (table.getIndexIdList().size() > 1 && !recorded.isPartitionSelectionComplete()) { Review Comment: Fixed: the index metadata of a pinned entry is now captured under the table read lock (released before the visible-version rpc), so a concurrent rollup/mv drop cannot abort HBO SHOW / DELETE STALE. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPlanStatisticsManager.java: ########## @@ -38,4 +177,431 @@ public HboPlanStatisticsProvider getHboPlanStatisticsProvider() { public HboPlanInfoProvider getHboPlanInfoProvider() { return hboPlanInfoProvider; } + + /** + * Inject (or overwrite) a pinned statistics entry applied unconditionally. + * @param fingerprint hbo fingerprint (simplified group struct info sha256) + * @param rows output row count that overrides the optimizer estimation + * @param structCanonical optional human-readable simplified struct info canonical string + */ + public void putPinnedPlanStatistics(String fingerprint, long rows, String structCanonical) { + putPinnedPlanStatistics(fingerprint, rows, PinnedType.EXACT, structCanonical, LiteralMode.NO_LITERAL); + } + + /** + * Inject (or overwrite) a pinned statistics entry. + * @param type how the injected row count is applied (unconditional vs filter-small guard) + * @param structCanonical optional human-readable simplified struct info canonical string + */ + public void putPinnedPlanStatistics(String fingerprint, long rows, PinnedType type, String structCanonical, + LiteralMode literalMode) { + putPinnedStatistics(fingerprint, rows, type, structCanonical, 0, literalMode); + } + + /** + * Inject (or overwrite) a pinned statistics entry. + * + * @param expansion fan-out factor, only for {@link PinnedType#JOIN_EXPANSION} entries + */ + private void putPinnedStatistics(String fingerprint, long rows, PinnedType type, String structCanonical, + double expansion, LiteralMode literalMode) { + // LRU bounded by Config.hbo_pinned_stats_cache_num; pinned entries are otherwise never + // expired automatically and are only removed by HBO DELETE STATISTICS or eviction + long createTimeMs = System.currentTimeMillis(); + synchronized (pinnedLoadLock) { + // the stored row is written first: when the write fails the statement fails and the entry + // is not pinned, so an entry which only lives in memory (and disappears on a restart) + // cannot be reported as set + if (persistenceEnabled() && !HboStatisticsStore.persist(fingerprint, rows, type, structCanonical, + expansion, literalMode, createTimeMs)) { + throw new IllegalStateException("failed to persist the hbo statistics entry " + fingerprint + + " into the internal table, see the FE log; the entry was not set"); + } + pinnedPlanStatistics.put(fingerprint, + new PinnedHboStatistics(fingerprint, rows, type, structCanonical, expansion, literalMode, + createTimeMs)); + if (type != PinnedType.JOIN_EXPANSION) { + // only row count entries are looked up by relation key: a join expansion entry is + // keyed by the join conditions, is rejected by getPinnedPlanStatistics, and its + // scanless canonical would turn the pre filter into a no-op for every group + registerRelationKey(structCanonical); + } + } + } + + /** + * The pinned row-count entry of a sub tree. Entries of another kind (a join expansion factor + * keyed by the join conditions) are never returned here, so the two read paths can not pick up + * each other's entry. + * + * <p>This is the planning hot path, so it only reads the in-memory cache: the persisted entries + * of this FE are loaded by {@link HboPinnedStatisticsLoader} in the background. Doing it here + * would run a DDL and a full table scan on a user query's planning thread, and the internal + * statement it runs would re-enter this manager (see the class javadoc). + */ + public Optional<PinnedHboStatistics> getPinnedPlanStatistics(String fingerprint) { + return Optional.ofNullable(pinnedPlanStatistics.getIfPresent(fingerprint)) + .filter(pinned -> !pinned.isExpansion()); + } + + /** + * Whether this FE holds any pinned entry at all (of any kind). The planning path looks an entry + * up by the hbo fingerprint of a plan node, and that fingerprint is expensive to build (a whole + * memo subtree traversal); an empty cache means every such lookup is a guaranteed miss, so the + * caller can skip building the fingerprint. Like {@link #getPinnedPlanStatistics(String)} this + * only reads the in-memory cache and never triggers the background loader. + */ + public boolean hasAnyPinnedStatistics() { + return !pinnedPlanStatistics.asMap().isEmpty(); + } + + /** + * Whether a pinned entry could possibly apply to a sub tree which reads exactly {@code + * relationKey} (see {@link GroupStructInfo.HboStructSummary#getRelationKey()}). + * + * <p>An entry is keyed by the fingerprint of its struct info, and two equal struct infos + * necessarily read the same tables, so an entry whose relation key differs can not match: the + * caller can skip building the (expensive) fingerprint. The index is an over approximation - + * keys are only ever added, so a removed or evicted entry may keep its key - which can only make + * a lookup happen, never suppress one. + */ + public boolean mayHavePinnedEntryForRelations(String relationKey) { + return pinnedEntriesWithoutRelationKey.get() > 0 || pinnedRelationKeys.contains(relationKey); + } + + /** Register the relation key of an entry's struct info in the lookup index. */ + private void registerRelationKey(String structCanonical) { + List<HboScanDescriptor> scans = HboScanDescriptor.parseAll(structCanonical); + if (scans.isEmpty()) { + // an entry which records no scan at all can not be filtered by a relation key + pinnedEntriesWithoutRelationKey.incrementAndGet(); + return; + } + List<String> tables = new ArrayList<>(scans.size()); + for (HboScanDescriptor scan : scans) { + tables.add(scan.getTable()); + } + Collections.sort(tables); + pinnedRelationKeys.add(String.join(",", tables)); + } + + /** + * Remove a pinned statistics entry. Learned entries of the same key (if any) are left + * untouched here; callers may invalidate them explicitly. + */ + public void removePinnedPlanStatistics(String fingerprint) { + synchronized (pinnedLoadLock) { + // the stored row is removed first: when that fails the statement fails and the in-memory + // entry stays authoritative, so a lazy load which reads the row later installs an entry + // which is still active (and which the user was told about) + deletePersistedPinnedPlanStatistics(fingerprint); + pinnedPlanStatistics.invalidate(fingerprint); + } + } + + /** + * Remove the entry only while it is still the one which was judged obsolete. + * + * <p>Two admin sessions can interleave: {@code HBO DELETE STALE STATISTICS} reads an entry and decides + * that its data state is gone, {@code HBO SET STATISTICS} replaces it with a fresh baseline, and a + * removal by fingerprint alone would then drop the *new* entry - the one the read side is about to + * apply. The identity check and the invalidation run under the same lock a SET takes, so a + * replacement cannot slip between them. + * + * @param judged the entry the caller decided about; only that very entry is removed + * @return whether the judged entry was still stored and has been removed + */ + public boolean removePinnedPlanStatisticsIfUnchanged(String fingerprint, PinnedHboStatistics judged) { + synchronized (pinnedLoadLock) { + if (pinnedPlanStatistics.getIfPresent(fingerprint) != judged) { + return false; + } + deletePersistedPinnedPlanStatistics(fingerprint); + pinnedPlanStatistics.invalidate(fingerprint); + return true; + } + } + + /** + * Remove the stored row. A failure is reported to the caller instead of being remembered: the + * in-memory entry is only removed after the row is gone, so a row the load reads later always + * describes an entry which is still active. + */ + private void deletePersistedPinnedPlanStatistics(String fingerprint) { + if (!persistenceEnabled()) { + return; + } + if (!HboStatisticsStore.delete(fingerprint)) { + throw new IllegalStateException("failed to remove the persisted hbo statistics entry " + + fingerprint + " from the internal table, see the FE log; the entry is still active"); + } + } + + /** + * Inject (or overwrite) the fan-out factor of a set of join equality conditions + * ({@code HBO SET STATISTICS ... TYPE=JOIN_EXPANSION}). It is a pinned entry like every other + * hbo statistics entry - same table, same lifecycle - only its value is a factor instead of a + * row count and its key is the join condition fingerprint. + * + * @param fingerprint sha256 of the canonical equality-condition set + * @param expansion measured fan-out factor relative to the larger input of the join (0.1 keeps + * 10% of the larger input, 1000 fans out to 1000 times it), greater than 0 + * @param condCanonical the canonical equality-condition set the factor was measured for + */ + public void putPinnedExpansionStatistics(String fingerprint, double expansion, String condCanonical) { + // a condition key carries its literals in both modes (a residual decides how many partner rows a + // row finds), and HBO SET STATISTICS stores the pasted text verbatim for such an entry, so the + // mode recorded here is the label of the line EXPLAIN prints rather than a folding instruction + putPinnedStatistics(fingerprint, 0, PinnedType.JOIN_EXPANSION, condCanonical, expansion, + LiteralMode.NO_LITERAL); + } + + /** + * The pinned join expansion entry of a set of equality conditions, if one was injected. + */ + public Optional<PinnedHboStatistics> getPinnedExpansion(String fingerprint) { + return Optional.ofNullable(pinnedPlanStatistics.getIfPresent(fingerprint)) + .filter(PinnedHboStatistics::isExpansion); + } + + /** + * Inject a learned entry (used by {@code HBO SET LEARNED STATISTICS}) so that the learned + * lookup path can be exercised without waiting for a real profile publish. The injected entry + * carries no input table statistics and therefore matches by fingerprint alone. + */ + public void putLearnedPlanStatistics(String fingerprint, long rows, String structCanonical) { + PlanStatistics planStatistics = new PlanStatistics(0, rows, rows, rows, rows, + rows, rows, rows, rows, 0, 0, 1); + RecentRunsPlanStatistics runs = new RecentRunsPlanStatistics( + Lists.newArrayList(new RecentRunsPlanStatisticsEntry(planStatistics, Lists.newArrayList()))); + hboPlanStatisticsProvider.putHboPlanStatsByFingerprint(fingerprint, runs); + if (structCanonical != null && !structCanonical.isEmpty()) { + learnedStructCanonical.put(fingerprint, structCanonical); + } else { + // the rows of the entry are replaced, so the label of the entry it replaces has to go + // away as well: otherwise SHOW reports the new count with the struct info, the state and + // the LIKE match of the previous SET + learnedStructCanonical.invalidate(fingerprint); + } + } + + /** The struct info a learned entry was injected with, when it was injected with one. */ + public Optional<String> getLearnedStructCanonical(String fingerprint) { + return Optional.ofNullable(learnedStructCanonical.getIfPresent(fingerprint)); + } + + /** Remove a learned entry by fingerprint. */ + public void removeLearnedPlanStatistics(String fingerprint) { + hboPlanStatisticsProvider.removeHboPlanStats(fingerprint); + learnedStructCanonical.invalidate(fingerprint); + } + + /** + * Remove every pinned entry which can never be applied again: an entry records the data state it + * was measured in (the baseline of the scans of its struct info), and once a table moved beyond + * {@code Config.hbo_row_count_change_ratio} (or was dropped) the entry is not applied any more + * (see {@link HboStructFreshness}); it only keeps a cache slot until it is removed here. Entries + * which are merely {@code drifted} (the data moved, but within the tolerance) are kept: they are + * still applied. + * + * @param olderThanMillis when non negative, entries whose state cannot be determined (their + * struct info records no comparable state, e.g. it prunes partitions or + * does not resolve to existing tables) are removed as well, but only when + * they are older than this many milliseconds; a negative value keeps them + * @return the number of removed entries and the number whose stored row could not be removed + */ + public int[] deleteStalePinnedPlanStatistics(long olderThanMillis) { + long now = System.currentTimeMillis(); + int removed = 0; + int failed = 0; + for (PinnedHboStatistics pinned : getAllPinnedPlanStatistics().values()) { + // an expansion entry is keyed by join conditions and carries no data state, so it can + // neither go stale nor be judged by age: OLDER_THAN must not silently drop it + if (pinned.isExpansion()) { + continue; + } + HboStructFreshness freshness = HboStructFreshness.of(pinned.getStructCanonical()); + boolean obsolete = freshness.isStale() + || (olderThanMillis >= 0 && freshness.isUnknown() + && now - pinned.getCreateTime() > olderThanMillis); + if (obsolete) { + // the entry may have been replaced by a concurrent set between the read above and this + // removal: only the entry which was judged obsolete is removed, never a fresh baseline + try { + if (removePinnedPlanStatisticsIfUnchanged(pinned.getFingerprint(), pinned)) { + removed++; + } + } catch (IllegalStateException e) { + // the stored row of this entry could not be removed: the entry stays active and is + // judged again by the next run, the other entries of this statement still go + failed++; + LOG.warn("failed to remove stale hbo statistics entry {}: {}", + pinned.getFingerprint(), e.getMessage()); + } + } + } + return new int[] {removed, failed}; + } + + /** + * Every pinned entry of this FE, after making sure the persisted ones have been loaded. + * + * <p>Unlike the planning path ({@link #getPinnedPlanStatistics}) this is only used by the admin + * statements ({@code HBO SHOW} / {@code HBO DELETE STALE STATISTICS}), which are expected to wait + * for the entries of the internal table and to report what is stored. Loading here cannot + * re-enter the way it could before the load moved to {@link HboPinnedStatisticsLoader}: the + * internal statement a load runs is planned by this FE, and its planning reads the pinned cache + * without triggering a load any more. + */ + public Map<String, PinnedHboStatistics> getAllPinnedPlanStatistics() { + ensurePinnedLoaded(); Review Comment: Fixed: ensurePinnedLoaded now reports whether the required load completed; the admin entry point (getAllPinnedPlanStatistics) raises an error while it is incomplete instead of presenting a partial cache. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboStatisticsStore.java: ########## @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.statistics.repository.ResultRow; +import org.apache.doris.statistics.util.StatisticsUtil; + +import com.google.common.collect.ImmutableList; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; + +/** + * Persistence of manually injected (pinned) hbo statistics into the internal database + * ({@code __internal_schema.hbo_statistics}), controlled by + * {@code Config.hbo_persist_pinned_to_internal_db}. + * + * <p>The in-memory {@link HboPlanStatisticsManager} stays authoritative on the read path; + * SET/DELETE are written through synchronously (best effort, failures only logged) and a FE + * loads the whole table into memory lazily on first use, so pinned entries survive a FE + * restart. All FEs share the table through the backend storage. + */ +public class HboStatisticsStore { + private static final Logger LOG = LogManager.getLogger(HboStatisticsStore.class); + + private static final String INTERNAL_DB = FeConstants.INTERNAL_DB_NAME; + private static final String TABLE = "hbo_statistics"; + private static final String FULL_QUALIFIED = + InternalCatalog.INTERNAL_CATALOG_NAME + "." + INTERNAL_DB + "." + TABLE; + /** max length (bytes) of the struct_info varchar column */ + private static final int STRUCT_MAX_BYTES = 65533; + + /** creation time of an entry, as a datetime(3) column like the other internal tables */ + private static final String CREATE_TIME_COLUMN = "create_time"; + /** fan-out factor of a PinnedType.JOIN_EXPANSION entry, NULL for a row count entry */ + private static final String EXPANSION_COLUMN = "expansion"; + /** which struct info the entry is keyed by (with_literal / no_literal) */ + private static final String LITERAL_MODE_COLUMN = "literal_mode"; + /** the columns this version writes and reads, checked once per FE (see verifySchema) */ + private static final List<String> COLUMNS = ImmutableList.of("fingerprint", "row_count", "stats_type", + LITERAL_MODE_COLUMN, "struct_info", EXPANSION_COLUMN, CREATE_TIME_COLUMN); + // the DDL and the schema check are done once per FE: the table can only change by a FE upgrade, + // and both cost a round trip to the internal table (a forwarded DDL on a follower FE) + private static volatile boolean tableCreated = false; + private static volatile boolean schemaVerified = false; + + private HboStatisticsStore() { + } + + /** + * Whether a canonical struct info fits the {@code struct_info} column without truncation. + * A truncated struct info would keep its fingerprint but lose its scan tokens (the baseline is + * printed after the conditions), which silently disables both the freshness verdict and the + * relation pre filter for that entry, so such an entry must not be accepted at all. + */ + public static boolean fitsStructColumn(String structCanonical) { + return structCanonical == null + || structCanonical.getBytes(java.nio.charset.StandardCharsets.UTF_8).length <= STRUCT_MAX_BYTES; + } + + /** Create the table if it does not exist, and check that its schema is the current one. */ + public static void ensureTable() throws Exception { + if (!tableCreated) { + StatisticsUtil.execUpdate(createDdl()); + tableCreated = true; + } + if (!schemaVerified) { + verifySchema(); + } + } + + /** + * Check that the table has every column this version reads and writes, by looking at its metadata + * in the FE catalog. A query would be cheaper to write but wrong to interpret: it fails both when + * the table has an old schema and when the internal schema is simply not ready yet, and the two + * need opposite reactions (drop the table vs wait). + * + * <p>A table created by an older FE version (e.g. with a {@code create_time_ms} column) cannot be + * written by this version, and because it is created with {@code CREATE TABLE IF NOT EXISTS} it + * stays broken; the check therefore logs the fix and forgets that the table was "created", so + * that the DDL runs again - and recreates the table - after the operator dropped it. Without + * that, persistence would stay dead until the FE restarts. + */ + private static void verifySchema() throws Exception { + Database db = Env.getCurrentEnv().getInternalCatalog().getDbNullable(INTERNAL_DB); + TableIf table = db == null ? null : db.getTableNullable(TABLE); + if (table == null) { + // the DDL succeeded but the table is not visible yet (a follower waiting for the journal, + // or an internal schema which is not ready): retry later, do not blame the table + throw new IllegalStateException("the internal table " + FULL_QUALIFIED + " is not visible yet"); + } + List<String> missing = new ArrayList<>(); + for (String column : COLUMNS) { + if (table.getColumn(column) == null) { + missing.add(column); + } + } + if (!missing.isEmpty()) { + tableCreated = false; + LOG.warn("the internal table {} has no {} column: it was created by an older FE version and" + + " has to be dropped so that it is recreated (DROP TABLE {})", + FULL_QUALIFIED, missing, FULL_QUALIFIED); + throw new IllegalStateException("the internal table " + FULL_QUALIFIED + " has an old schema"); + } + schemaVerified = true; + } + + /** + * Upsert one pinned entry (UNIQUE KEY fingerprint replaces on conflict). + */ + public static boolean persist(String fingerprint, long rows, + HboPlanStatisticsManager.PinnedType type, String structCanonical, double expansion, + HboPlanStatisticsManager.LiteralMode literalMode, long createTimeMs) { + try { + ensureTable(); + HboPlanStatisticsManager.PinnedType statsType = type == null + ? HboPlanStatisticsManager.PinnedType.EXACT : type; + // an empty struct_info records no data state: the freshness comparison reports unknown for such + // an entry and the pinned lookup index has no relation key for it, which is why the pinned scope + // of HBO SET requires the canonical string (see HboStatisticsCommand) + String struct = truncateUtf8(structCanonical == null ? "" : structCanonical, STRUCT_MAX_BYTES); + // the fingerprint kind is only known after the entry was applied once, so the stored + // value stays UNKNOWN (it is reported by HBO SHOW from the in-memory entry) + String sql = "INSERT INTO " + FULL_QUALIFIED + + " (`fingerprint`, `row_count`, `stats_type`, `" + LITERAL_MODE_COLUMN + "`, `struct_info`," + + " `" + EXPANSION_COLUMN + "`, `" + CREATE_TIME_COLUMN + "`) VALUES ('" + + StatisticsUtil.escapeSQL(fingerprint) + "', " + rows + ", '" + + StatisticsUtil.escapeSQL(statsType.name().toLowerCase(java.util.Locale.ROOT)) + "', '" + + StatisticsUtil.escapeSQL((literalMode == null + ? HboPlanStatisticsManager.LiteralMode.NO_LITERAL : literalMode) + .name().toLowerCase(java.util.Locale.ROOT)) + + "', '" + StatisticsUtil.escapeSQL(struct) + "', " Review Comment: Fixed: the internal session of execUpdate clears NO_BACKSLASH_ESCAPES, so the values written with escapeSQL are parsed the way they were produced and a structure with a backslash keeps its bytes across a restart. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
