This is an automated email from the ASF dual-hosted git repository. gortiz pushed a commit to branch cbo-1-stats-contracts-and-stores in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 3a1b835cc591e554b6ac2248a0d4df7ba252d7ab Author: Gonzalo Ortiz <[email protected]> AuthorDate: Thu Aug 27 20:02:17 2026 +0200 Add statistics contracts for cost-based optimization Defines what a Pinot statistic is, before anything produces or consumes one: TableStatistics and ColumnStatistics as the values, StatConfidence as the trust tier attached to each one, StatsStore as broker-local persistence over per-segment rows, and ColumnStatsSource as where per-column statistics are fetched from. Confidence is per statistic rather than per table, so a source that can only estimate some values does not have to devalue the rest. Consumers are expected to treat a low-confidence statistic as absent instead of trusting it, which is what keeps table types with biased raw counts (upsert, dedup, consuming segments) on today's behavior rather than producing confidently wrong plans. StatsAggregations carries the rollup semantics every store must share, so two implementations cannot disagree about what the same stored rows mean: time overlap interpolation, document-weighted averages that exclude the "unknown" sentinel rather than averaging it in as a measurement, and min/max folded under the ordering the column actually has. That ordering is recorded per row as a ColumnValueType rather than guessed from the text, because guessing is wrong in both directions: a string column holding "9" and "10" orders lexically in Pinot but would compare numerically, and a long beyond 2^53 loses digits as a double -- in the direction that narrows the range, which would exclude rows that exist. When the ordering is unknown or segments disagree about it, both bounds are reported as absent: a bound folded under two different orderings is neither a true minimum nor a true maximum, and there is no honest way to describe it as merely untrusted. The store contract includes getTables(), so a caller can find tables it no longer serves. Per-table cleanup is otherwise driven by an event -- a routing entry being removed -- which a broker cannot observe for a table dropped while it was down. --- CLAUDE.md | 6 + .../query/planner/spi/stats/ColumnStatistics.java | 180 ++++++++++++++++++++ .../spi/stats/ColumnStatsFetchException.java | 47 ++++++ .../query/planner/spi/stats/ColumnStatsSource.java | 56 ++++++ .../query/planner/spi/stats/ColumnValueType.java | 107 ++++++++++++ .../planner/spi/stats/SegmentColumnStatsRow.java | 113 +++++++++++++ .../query/planner/spi/stats/SegmentStatsRow.java | 34 ++++ .../query/planner/spi/stats/StatConfidence.java | 47 ++++++ .../query/planner/spi/stats/StatsAggregations.java | 187 +++++++++++++++++++++ .../pinot/query/planner/spi/stats/StatsStore.java | 153 +++++++++++++++++ .../planner/spi/stats/StatsStoreException.java | 44 +++++ .../planner/spi/stats/StatsStoreProvider.java | 45 +++++ .../query/planner/spi/stats/TableStatistics.java | 116 +++++++++++++ .../planner/spi/stats/ColumnValueTypeTest.java | 106 ++++++++++++ .../planner/spi/stats/StatsAggregationsTest.java | 148 ++++++++++++++++ 15 files changed, 1389 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 046bd537a09..9eae2dafd4f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -111,6 +111,12 @@ Apache Pinot is a real-time distributed OLAP datastore for low-latency analytics - Keep Apache 2.0 license headers on all new source files. - Preserve backward compatibility across mixed-version broker/server/controller. - Prefer imports over fully qualified class names (e.g., use `import com.foo.Bar` and refer to `Bar`, not `com.foo.Bar` inline). +- Prefer records for immutable value carriers, especially in new APIs. A record states the + intent (data, not behaviour), gives correct `equals`/`hashCode`/`toString` for free, and cannot + drift out of sync with its fields. Use a class only when the type genuinely needs mutability, + inheritance, or a non-trivial identity contract. Note that a record does not excuse a dangerous + constructor: when a record has several components of the same type in a row, add a builder so + call sites name what they pass — a record and a builder compose fine. - Prefer `List.of()`, `Set.of()`, and `Map.of()` for non-null immutable collection literals. Checkstyle blocks `Collections.emptyList()`, `Collections.emptySet()`, and `Collections.emptyMap()`; use `List.of()`, `Set.of()`, and `Map.of()` instead. Do not add blanket bans for `Collections.singleton*`; use them only when an element/key/value diff --git a/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/ColumnStatistics.java b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/ColumnStatistics.java new file mode 100644 index 00000000000..6377258252f --- /dev/null +++ b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/ColumnStatistics.java @@ -0,0 +1,180 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.spi.stats; + +import java.util.Objects; +import javax.annotation.Nullable; + + +/// Immutable per-column statistics for a table, used by the cost-based query planner. +/// +/// Instances are created via [#builder()]. Unknown numeric fields are represented +/// by `-1`. Min/max values may be `null` when unknown. +/// +/// Thread-safety: immutable; safe for concurrent access. +public class ColumnStatistics { + private final String _columnName; + private final long _ndv; + private final StatConfidence _ndvConfidence; + @Nullable + private final Comparable<?> _minValue; + @Nullable + private final Comparable<?> _maxValue; + private final boolean _minTrusted; + private final double _avgBytesPerValue; + private final double _nullFraction; + + private ColumnStatistics(Builder builder) { + _columnName = builder._columnName; + _ndv = builder._ndv; + _ndvConfidence = builder._ndvConfidence; + _minValue = builder._minValue; + _maxValue = builder._maxValue; + _minTrusted = builder._minTrusted; + _avgBytesPerValue = builder._avgBytesPerValue; + _nullFraction = builder._nullFraction; + } + + /// Returns a new [Builder] for constructing [ColumnStatistics] instances. + public static Builder builder() { + return new Builder(); + } + + /// Returns the name of the column these statistics describe. + public String getColumnName() { + return _columnName; + } + + /// Returns the estimated number of distinct values (NDV) for the column, or `-1` if unknown. + public long getNdv() { + return _ndv; + } + + /// Returns the confidence level of the [#getNdv()] value. + public StatConfidence getNdvConfidence() { + return _ndvConfidence; + } + + /// Returns the minimum observed value for the column, or `null` if unknown. + @Nullable + public Comparable<?> getMinValue() { + return _minValue; + } + + /// Returns the maximum observed value for the column, or `null` if unknown. + @Nullable + public Comparable<?> getMaxValue() { + return _maxValue; + } + + /// Returns `false` when the minimum value is polluted by the numeric null-sentinel default + /// (i.e. the column is nullable and its stored min equals the type's minimum representable value). + /// When `false`, estimation may assume the range `[-max..max]`, and segment pruning + /// must never use this bound. + public boolean isMinTrusted() { + return _minTrusted; + } + + /// Returns the average number of bytes per stored value for the column, or `-1` if unknown. + public double getAvgBytesPerValue() { + return _avgBytesPerValue; + } + + /// Returns the fraction of rows where the column value is null (in the range `[0.0, 1.0]`), + /// or `-1` if unknown. + public double getNullFraction() { + return _nullFraction; + } + + /// Builder for [ColumnStatistics]. + /// + /// [#columnName(String)] is required; every other field is optional. Default values: string + /// fields default to `null`, numeric fields to `-1` (unknown), confidence fields to + /// [StatConfidence#UNKNOWN], and `minTrusted` defaults to `true`. + /// + /// Thread-safety: not thread-safe; use from a single thread. + public static class Builder { + private String _columnName; + private long _ndv = -1; + private StatConfidence _ndvConfidence = StatConfidence.UNKNOWN; + @Nullable + private Comparable<?> _minValue; + @Nullable + private Comparable<?> _maxValue; + private boolean _minTrusted = true; + private double _avgBytesPerValue = -1; + private double _nullFraction = -1; + + private Builder() { + } + + /// Sets the column name. + public Builder columnName(String columnName) { + _columnName = columnName; + return this; + } + + /// Sets the number of distinct values (NDV) and its confidence level. + public Builder ndv(long ndv, StatConfidence confidence) { + _ndv = ndv; + _ndvConfidence = confidence; + return this; + } + + /// Sets the minimum observed value. + public Builder minValue(@Nullable Comparable<?> minValue) { + _minValue = minValue; + return this; + } + + /// Sets the maximum observed value. + public Builder maxValue(@Nullable Comparable<?> maxValue) { + _maxValue = maxValue; + return this; + } + + /// Sets whether the minimum value is trustworthy (i.e. not polluted by a null-sentinel + /// default). + public Builder minTrusted(boolean minTrusted) { + _minTrusted = minTrusted; + return this; + } + + /// Sets the average number of bytes per stored value. + public Builder avgBytesPerValue(double avgBytesPerValue) { + _avgBytesPerValue = avgBytesPerValue; + return this; + } + + /// Sets the null fraction (proportion of null values in `[0.0, 1.0]`). + public Builder nullFraction(double nullFraction) { + _nullFraction = nullFraction; + return this; + } + + /// Builds the immutable [ColumnStatistics] instance. + /// + /// @throws NullPointerException if no column name was set; [#getColumnName()] is declared + /// non-null, so a nameless instance would hand callers a null they cannot see coming. + public ColumnStatistics build() { + Objects.requireNonNull(_columnName, "columnName must be set"); + return new ColumnStatistics(this); + } + } +} diff --git a/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/ColumnStatsFetchException.java b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/ColumnStatsFetchException.java new file mode 100644 index 00000000000..ba47ae0ccf6 --- /dev/null +++ b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/ColumnStatsFetchException.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.spi.stats; + + +/// Checked exception thrown by [ColumnStatsSource] when column statistics cannot be fetched and no +/// partial result can be returned. +/// +/// Throwing is the last resort: a source that obtained statistics for some segments should return +/// them and omit the rest, since a partial result still improves estimates. Callers must treat this +/// as "no new statistics this round" and degrade to whatever is already stored — collection failures +/// must never fail a query. +/// +/// Thread-safety: exception objects are not shared; no concurrency requirements. +public class ColumnStatsFetchException extends Exception { + + /// Constructs a new [ColumnStatsFetchException] with the given message. + /// + /// @param message description of the error + public ColumnStatsFetchException(String message) { + super(message); + } + + /// Constructs a new [ColumnStatsFetchException] with the given message and cause. + /// + /// @param message description of the error + /// @param cause the underlying cause + public ColumnStatsFetchException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/ColumnStatsSource.java b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/ColumnStatsSource.java new file mode 100644 index 00000000000..056158e9981 --- /dev/null +++ b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/ColumnStatsSource.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.spi.stats; + +import java.util.List; +import java.util.Map; +import java.util.Set; + + +/// Source of per-segment column statistics fetched on behalf of the broker: the acquisition side +/// of the statistics subsystem, paired with [StatsStore] (where statistics are kept) and +/// [PinotStatisticsProvider] (how the planner reads them). +/// +/// This is an extension point on purpose. A server fan-out, a push feed and a vendor metadata +/// service have very different cost profiles, so the choice of source must not be baked into the +/// collection logic. +/// +/// Implementations own their own bounding strategy (rate limits, jitter, debounce) and are +/// responsible for not overwhelming downstream services. +/// +/// The result is keyed by segment name; segments for which statistics could not be obtained +/// may be absent from the returned map. +/// +/// Thread-safety: implementations must be thread-safe. +public interface ColumnStatsSource { + + /// Fetches per-column statistics for the specified segments of the given table. + /// + /// Segments for which statistics are unavailable may be absent from the result map. + /// Implementations may return a partial result on partial failure. + /// + /// @param tableNameWithType fully-qualified table name including type suffix + /// @param segmentNames names of the segments for which statistics are requested + /// @return map from segment name to a list of per-column statistics rows; missing segments are + /// absent from the map + /// @throws ColumnStatsFetchException if fetching fails and no partial result can be returned + Map<String, List<SegmentColumnStatsRow>> fetchColumnStats(String tableNameWithType, + Set<String> segmentNames) + throws ColumnStatsFetchException; +} diff --git a/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/ColumnValueType.java b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/ColumnValueType.java new file mode 100644 index 00000000000..b8f58dbc515 --- /dev/null +++ b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/ColumnValueType.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.spi.stats; + +import java.math.BigDecimal; +import javax.annotation.Nullable; + + +/// How a column's min/max values, which are stored as text, must be ordered and deserialized. +/// +/// The type has to be recorded with the values because it cannot be recovered from them. Guessing +/// "numeric if it parses" is wrong in both directions: a STRING column holding `"9"` and `"10"` +/// orders lexically in Pinot but would compare numerically, and a LONG beyond 2^53 loses precision +/// as a `double` — in the direction that narrows the range, which would exclude rows that exist. +/// +/// This enum deliberately mirrors only the ordering classes the statistics layer needs rather than +/// Pinot's full `DataType`: this module does not depend on `pinot-spi` at compile scope, and the +/// statistics layer only ever needs to know how to order a value, not how it is encoded. Producers +/// map their column type onto it. +public enum ColumnValueType { + /// Exact integral ordering; parsed as [Long] so values beyond 2^53 keep every digit. + LONG, + /// Floating-point ordering. + DOUBLE, + /// Arbitrary-precision numeric ordering. + BIG_DECIMAL, + /// Lexical ordering, matching how Pinot orders string columns. + STRING; + + /// Resolves a persisted type name, returning `null` for any name this build does not know. + /// + /// Deliberately not [#valueOf(String)]: the name comes from a store that can outlive the process + /// that wrote it, so a broker reading a file written by a newer build -- an aborted rolling + /// upgrade, say -- would otherwise throw [IllegalArgumentException] out of the query-planning + /// path. `null` is already the documented "ordering unknown" value, which degrades to untrusted + /// bounds, so an unrecognized name costs precision rather than the query. + @Nullable + public static ColumnValueType fromName(@Nullable String name) { + if (name == null) { + return null; + } + for (ColumnValueType type : values()) { + if (type.name().equals(name)) { + return type; + } + } + return null; + } + + /// Orders two stored values of this type. Values that cannot be parsed as this type fall back to + /// lexical order, so a malformed row degrades rather than throwing on the planning path. + public int compare(String a, String b) { + try { + switch (this) { + case LONG: + return Long.compare(Long.parseLong(a), Long.parseLong(b)); + case DOUBLE: + return Double.compare(Double.parseDouble(a), Double.parseDouble(b)); + case BIG_DECIMAL: + return new BigDecimal(a).compareTo(new BigDecimal(b)); + default: + return a.compareTo(b); + } + } catch (NumberFormatException e) { + return a.compareTo(b); + } + } + + /// Deserializes a stored value into the [Comparable] a consumer expects for this type, or the + /// raw [String] when it cannot be parsed. + @Nullable + public Comparable<?> toComparable(@Nullable String value) { + if (value == null) { + return null; + } + try { + switch (this) { + case LONG: + return Long.parseLong(value); + case DOUBLE: + return Double.parseDouble(value); + case BIG_DECIMAL: + return new BigDecimal(value); + default: + return value; + } + } catch (NumberFormatException e) { + return value; + } + } +} diff --git a/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/SegmentColumnStatsRow.java b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/SegmentColumnStatsRow.java new file mode 100644 index 00000000000..5260d2f2135 --- /dev/null +++ b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/SegmentColumnStatsRow.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.spi.stats; + +import javax.annotation.Nullable; + +/// Per-column statistics for a single segment, as persisted in the broker-local [StatsStore]. +/// +/// Min/max values are serialized as strings so heterogeneous column types share one schema; the +/// ordering to compare them under is carried separately by `valueType`, because it cannot be +/// recovered from the text. Unknown numeric fields are represented by `-1`. +/// +/// Prefer [#builder()] over the canonical constructor: the components include two adjacent +/// strings (min/max) and two adjacent doubles (avg bytes / null fraction), so a transposition +/// compiles silently and corrupts statistics. +/// +/// @param segmentName the segment name +/// @param columnName the column name +/// @param ndv number of distinct values, or `-1` if unknown +/// @param minValue minimum value as text, or `null` if unknown +/// @param maxValue maximum value as text, or `null` if unknown +/// @param minTrusted `false` when the minimum may be polluted by a null sentinel +/// @param avgBytesPerValue average encoded size per value, or `-1` if unknown +/// @param nullFraction fraction of null values in `[0, 1]`, or `-1` if unknown +/// @param valueType how min/max must be ordered, or `null` if unknown +public record SegmentColumnStatsRow(String segmentName, String columnName, long ndv, @Nullable String minValue, + @Nullable String maxValue, boolean minTrusted, double avgBytesPerValue, + double nullFraction, @Nullable ColumnValueType valueType) { + + public static Builder builder() { + return new Builder(); + } + + /// Names each component at the call site, so no pair of same-typed fields can be swapped + /// silently. Unset numeric fields default to the `-1` unknown sentinel. + /// + /// Thread-safety: not thread-safe; use from a single thread. + public static final class Builder { + private String _segmentName; + private String _columnName; + private long _ndv = -1; + @Nullable + private String _minValue; + @Nullable + private String _maxValue; + private boolean _minTrusted = true; + private double _avgBytesPerValue = -1; + private double _nullFraction = -1; + @Nullable + private ColumnValueType _valueType; + + public Builder segmentName(String segmentName) { + _segmentName = segmentName; + return this; + } + + public Builder columnName(String columnName) { + _columnName = columnName; + return this; + } + + public Builder ndv(long ndv) { + _ndv = ndv; + return this; + } + + /// Bounds and the ordering they must be compared under, set together because a bound whose + /// ordering is unknown cannot be used. + public Builder bounds(@Nullable String minValue, @Nullable String maxValue, + @Nullable ColumnValueType valueType) { + _minValue = minValue; + _maxValue = maxValue; + _valueType = valueType; + return this; + } + + public Builder minTrusted(boolean minTrusted) { + _minTrusted = minTrusted; + return this; + } + + public Builder avgBytesPerValue(double avgBytesPerValue) { + _avgBytesPerValue = avgBytesPerValue; + return this; + } + + public Builder nullFraction(double nullFraction) { + _nullFraction = nullFraction; + return this; + } + + public SegmentColumnStatsRow build() { + return new SegmentColumnStatsRow(_segmentName, _columnName, _ndv, _minValue, _maxValue, _minTrusted, + _avgBytesPerValue, _nullFraction, _valueType); + } + } +} diff --git a/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/SegmentStatsRow.java b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/SegmentStatsRow.java new file mode 100644 index 00000000000..6d371844e30 --- /dev/null +++ b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/SegmentStatsRow.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.spi.stats; + +/// Aggregate statistics for a single segment, as persisted in the broker-local [StatsStore]. +/// +/// Unknown numeric fields are represented by `-1`. +/// +/// @param segmentName the segment name +/// @param crc CRC checksum of the segment, or `-1` if unknown +/// @param totalDocs total documents in the segment, or `-1` if unknown +/// @param sizeBytes on-disk size in bytes, or `-1` if unknown +/// @param startTimeMs start of the segment's time range in epoch millis, or `-1` if unknown +/// @param endTimeMs end of the segment's time range in epoch millis, or `-1` if unknown +/// @param consuming `true` for a consuming (REALTIME IN_PROGRESS) segment +public record SegmentStatsRow(String segmentName, long crc, long totalDocs, long sizeBytes, long startTimeMs, + long endTimeMs, boolean consuming) { +} diff --git a/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/StatConfidence.java b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/StatConfidence.java new file mode 100644 index 00000000000..60dd0c45942 --- /dev/null +++ b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/StatConfidence.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.spi.stats; + + +/// Indicates how trustworthy a statistics value is for cost-based query planning. +/// +/// Callers should treat [#LOW] statistics the same as [#UNKNOWN] for +/// cost-based decisions because LOW values are known to be systematically biased. +/// +/// This enum is append-only — new confidence levels may be added without breaking +/// code compiled against an older version. Existing constants must never be reordered +/// or removed. +/// +/// Thread-safety: enum constants are inherently thread-safe. +public enum StatConfidence { + /// Derived from authoritative metadata (e.g. sum of committed segments' totalDocs for an + /// OFFLINE table). + EXACT, + + /// Derived via approximation (e.g. clamped NDV merge, interpolated time ranges). + ESTIMATED, + + /// Known to be systematically biased (e.g. upsert tables where physical doc count over-counts + /// logical rows; tables with consuming segments). Planner must treat LOW like absent stats for + /// cost-based decisions. + LOW, + + /// No information available. + UNKNOWN +} diff --git a/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/StatsAggregations.java b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/StatsAggregations.java new file mode 100644 index 00000000000..d204c4b9c4f --- /dev/null +++ b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/StatsAggregations.java @@ -0,0 +1,187 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.spi.stats; + +import javax.annotation.Nullable; + + +/// Aggregation semantics shared by every [StatsStore] implementation. +/// +/// These rules define what the stored per-segment rows *mean*, so they must not be re-derived per +/// implementation: two stores that disagree here would make the optimizer behave differently +/// depending on which store an operator configured. They live beside the contract, and are public, +/// so an implementation outside this package can reuse them rather than reinventing them. +/// +/// Thread-safety: stateless; all methods are pure. +public final class StatsAggregations { + + private StatsAggregations() { + } + + /// Returns how many of a segment's `docs` fall in the half-open query range + /// `[startMs, endMs)`, given the segment's own inclusive time range `[segStart, segEnd]`. + /// + /// - Unknown segment times (the `-1` sentinel on either bound) count in full: the segment cannot + /// be excluded, and over-counting is the conservative direction for an estimate. + /// - No overlap contributes 0. + /// - Full containment contributes every doc. + /// - Partial overlap is interpolated linearly over the segment's duration. + /// - A zero-length segment counts in full when its single point lies in range. + public static long overlapRows(long docs, long segStart, long segEnd, long startMs, long endMs) { + if (segStart == -1 || segEnd == -1) { + return docs; + } + if (segEnd <= startMs || segStart >= endMs) { + return 0; + } + if (segStart >= startMs && segEnd <= endMs) { + return docs; + } + long segDuration = segEnd - segStart; + if (segDuration <= 0) { + return segStart >= startMs && segStart < endMs ? docs : 0; + } + long overlapStart = Math.max(startMs, segStart); + long overlapEnd = Math.min(endMs, segEnd); + double fraction = (double) (overlapEnd - overlapStart) / segDuration; + return Math.round(docs * fraction); + } + + /// Accumulates per-segment column rows into a single [ColumnStatistics], applying the rules that + /// define what those stored rows mean. + /// + /// Every store must fold rows through this, so that the estimate a query gets cannot depend on + /// which store an operator configured. + /// + /// Thread-safety: not thread-safe; use one accumulator per aggregation. + public static final class ColumnStatsAccumulator { + private long _maxNdv = -1; + private boolean _anyUntrustedMin; + private long _totalDocs; + private double _weightedAvgBytes; + /// Documents behind [#_weightedAvgBytes]. Tracked separately from [#_totalDocs] because rows + /// carrying the "unknown" sentinel contribute no weight, so dividing by the full document + /// count would understate the average. + private long _avgBytesDocs; + private double _weightedNullFraction; + /// Documents behind [#_weightedNullFraction]; see [#_avgBytesDocs]. + private long _nullFractionDocs; + @Nullable + private String _min; + @Nullable + private String _max; + @Nullable + private ColumnValueType _valueType; + private boolean _typeConflict; + private boolean _empty = true; + + /// Adds one segment's row for this column, weighted by that segment's document count. + public void add(long segmentDocs, SegmentColumnStatsRow row) { + _empty = false; + _maxNdv = Math.max(_maxNdv, row.ndv()); + if (!row.minTrusted()) { + _anyUntrustedMin = true; + } + _totalDocs += segmentDocs; + // Both fields reserve a negative value for "unknown" (see SegmentColumnStatsRow). Weighting + // that sentinel in as if it were a measurement produces a nonsense average -- a mix of an + // unknown row and a known 0.2 null fraction would yield a negative fraction, which is + // neither a legal fraction nor the sentinel a consumer tests for. + double avgBytes = row.avgBytesPerValue(); + if (avgBytes >= 0) { + _weightedAvgBytes += avgBytes * segmentDocs; + _avgBytesDocs += segmentDocs; + } + double nullFraction = row.nullFraction(); + if (nullFraction >= 0) { + _weightedNullFraction += nullFraction * segmentDocs; + _nullFractionDocs += segmentDocs; + } + + ColumnValueType rowType = row.valueType(); + if (rowType == null) { + // No recorded ordering: the text cannot tell us how to compare, so stop trusting bounds + // rather than guessing one. + _typeConflict = true; + } else if (_valueType == null) { + _valueType = rowType; + } else if (_valueType != rowType) { + // Segments disagreeing about a column's type means one of them is stale; ordering across + // them is undefined. + _typeConflict = true; + } + + // Once the ordering is in doubt the bounds are discarded wholesale by build(), so there is + // nothing to gain by folding further rows into them under a guessed ordering. + if (!_typeConflict && _valueType != null) { + _min = minOf(_min, row.minValue(), _valueType); + _max = maxOf(_max, row.maxValue(), _valueType); + } + } + + /// Returns `true` when no row was added, in which case the caller reports "no statistics" + /// rather than an empty aggregate. + public boolean isEmpty() { + return _empty; + } + + public ColumnStatistics build(String columnName) { + ColumnValueType effectiveType = _typeConflict || _valueType == null ? null : _valueType; + return ColumnStatistics.builder() + .columnName(columnName) + // Bounds folded without a known ordering are neither the true minimum nor the true + // maximum under any ordering, so they are reported as absent rather than as untrusted: + // isMinTrusted says nothing about the maximum, and a consumer following its documented + // remedy would build a range out of an equally unreliable bound. + .minValue(effectiveType == null ? null : effectiveType.toComparable(_min)) + .maxValue(effectiveType == null ? null : effectiveType.toComparable(_max)) + .ndv(_maxNdv, StatConfidence.ESTIMATED) + .minTrusted(!_anyUntrustedMin && effectiveType != null) + .avgBytesPerValue(_avgBytesDocs > 0 ? _weightedAvgBytes / _avgBytesDocs : -1) + .nullFraction(_nullFractionDocs > 0 ? _weightedNullFraction / _nullFractionDocs : -1) + .build(); + } + } + + /// Returns the smaller of two stored values under `type`; `null` means unknown, so the other + /// value wins. + @Nullable + public static String minOf(@Nullable String a, @Nullable String b, ColumnValueType type) { + if (a == null) { + return b; + } + if (b == null) { + return a; + } + return type.compare(a, b) <= 0 ? a : b; + } + + /// Returns the larger of two stored values under `type`; `null` means unknown, so the other + /// value wins. + @Nullable + public static String maxOf(@Nullable String a, @Nullable String b, ColumnValueType type) { + if (a == null) { + return b; + } + if (b == null) { + return a; + } + return type.compare(a, b) >= 0 ? a : b; + } +} diff --git a/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/StatsStore.java b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/StatsStore.java new file mode 100644 index 00000000000..0326e98f2cd --- /dev/null +++ b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/StatsStore.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.spi.stats; + +import java.io.Closeable; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; +import java.util.Set; +import javax.annotation.Nullable; + + +/// Broker-local persistence for table and segment statistics used by the cost-based query planner. +/// +/// Durability is implementation-defined: a durable store survives broker restarts warm, while a +/// non-durable store restarts empty and relies on the caller's reconciliation path +/// ([#getSegmentCrcs]) to re-collect, trading restart re-collection cost for zero disk usage. +/// +/// Threading: writes arrive from a pool that processes segment-assignment changes table by table, +/// so calls are serialized per `tableNameWithType` but NOT across tables — implementations must be +/// safe for concurrent writers to different tables, and for readers at any time. READ failures must +/// be cheap to detect so callers can degrade to a no-stats path; callers must never fail a query +/// because of a store error. +/// +/// Lifecycle: call [#init()] once before any other method; call [#close()] to release resources. +/// +/// Thread-safety: see the threading note above — concurrent readers plus concurrent writers to +/// distinct tables. +public interface StatsStore extends Closeable { + + /// Opens the store and migrates the schema if necessary. + /// + /// On unrecoverable corruption, implementations must drop and recreate an empty store rather + /// than propagating an error. + /// + /// @throws StatsStoreException if the store cannot be opened or initialized + void init() + throws StatsStoreException; + + /// Inserts or updates segment-level statistics for the given table. + /// + /// @param tableNameWithType fully-qualified table name including type suffix + /// @param rows segment statistics rows to upsert + /// @throws StatsStoreException if the write fails + void upsertSegmentStats(String tableNameWithType, List<SegmentStatsRow> rows) + throws StatsStoreException; + + /// Inserts or updates per-column statistics for individual segments of the given table. + /// + /// @param tableNameWithType fully-qualified table name including type suffix + /// @param rows segment column statistics rows to upsert + /// @throws StatsStoreException if the write fails + void upsertSegmentColumnStats(String tableNameWithType, List<SegmentColumnStatsRow> rows) + throws StatsStoreException; + + /// Removes all stored statistics for the specified segments of the given table. + /// + /// @param tableNameWithType fully-qualified table name including type suffix + /// @param segmentNames names of segments to remove + /// @throws StatsStoreException if the removal fails + void removeSegments(String tableNameWithType, Collection<String> segmentNames) + throws StatsStoreException; + + /// Returns a map from segment name to CRC for all segments of the given table. + /// + /// Used for restart reconciliation to detect stale or missing entries. + /// + /// @param tableNameWithType fully-qualified table name including type suffix + /// @return map of segment name to CRC; empty map if no segments are stored + /// @throws StatsStoreException if the read fails + Map<String, Long> getSegmentCrcs(String tableNameWithType) + throws StatsStoreException; + + /// Returns aggregated table-level statistics derived from all non-consuming segments, or + /// `null` if no statistics are available. + /// + /// @param tableNameWithType fully-qualified table name including type suffix + /// @throws StatsStoreException if the read fails + @Nullable + TableStatistics getTableStats(String tableNameWithType) + throws StatsStoreException; + + /// Returns per-column statistics aggregated across all segments for the given table and column, + /// or `null` if no statistics are available. + /// + /// @param tableNameWithType fully-qualified table name including type suffix + /// @param columnName name of the column + /// @throws StatsStoreException if the read fails + @Nullable + ColumnStatistics getColumnStats(String tableNameWithType, String columnName) + throws StatsStoreException; + + /// Returns an estimate of the number of rows whose time column falls in the half-open interval + /// `[startMs, endMs)`, or an empty optional if the estimate cannot be produced. + /// + /// @param tableNameWithType fully-qualified table name including type suffix + /// @param startMs start of the time range, inclusive, in epoch milliseconds + /// @param endMs end of the time range, exclusive, in epoch milliseconds + /// @throws StatsStoreException if the read fails + OptionalLong estimateRowsInTimeRange(String tableNameWithType, long startMs, long endMs) + throws StatsStoreException; + + /// Returns every table this store currently holds statistics for. + /// + /// Exists so a caller can find tables it no longer serves: per-table cleanup is otherwise driven + /// by an event (a routing entry being removed), which a broker cannot observe for a table dropped + /// while it was down. Without this the rows of such a table would stay in a durable store forever. + /// + /// @return fully-qualified table names, including type suffix; empty if the store holds nothing + /// @throws StatsStoreException if the read fails + Set<String> getTables() + throws StatsStoreException; + + /// Removes all stored statistics for the given table. + /// + /// @param tableNameWithType fully-qualified table name including type suffix + /// @throws StatsStoreException if the purge fails + void purgeTable(String tableNameWithType) + throws StatsStoreException; + + /// Removes all stored statistics for all tables. + /// + /// @throws StatsStoreException if the purge fails + void purgeAll() + throws StatsStoreException; + + /// Returns `true` if the given table has at least one consuming (REALTIME IN_PROGRESS) + /// segment in the store. + /// + /// Used to detect whether realtime row counts may undercount fresh, un-committed data. + /// + /// @param tableNameWithType fully-qualified table name including type suffix + /// @throws StatsStoreException if the read fails + boolean hasConsumingSegments(String tableNameWithType) + throws StatsStoreException; +} diff --git a/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/StatsStoreException.java b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/StatsStoreException.java new file mode 100644 index 00000000000..7c9b2b6fb77 --- /dev/null +++ b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/StatsStoreException.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.spi.stats; + + +/// Checked exception thrown by [StatsStore] operations when a storage error occurs. +/// +/// Callers of [StatsStore] should catch this exception and degrade gracefully to a +/// no-stats path rather than failing the query. +/// +/// Thread-safety: exception objects are not shared; no concurrency requirements. +public class StatsStoreException extends Exception { + + /// Constructs a new [StatsStoreException] with the given message. + /// + /// @param message description of the error + public StatsStoreException(String message) { + super(message); + } + + /// Constructs a new [StatsStoreException] with the given message and cause. + /// + /// @param message description of the error + /// @param cause the underlying cause + public StatsStoreException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/StatsStoreProvider.java b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/StatsStoreProvider.java new file mode 100644 index 00000000000..12802288640 --- /dev/null +++ b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/StatsStoreProvider.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.spi.stats; + +import java.util.Map; + + +/// Creates a [StatsStore] for a configured name, discovered through [java.util.ServiceLoader]. +/// +/// Implementations declare their own name, so operators configure a stable identifier rather than a +/// class name: renaming or moving the implementation must not invalidate an operator's +/// configuration. Register one by listing it in +/// `META-INF/services/org.apache.pinot.query.planner.spi.stats.StatsStoreProvider`. +/// +/// Thread-safety: providers are discovered once and shared, so implementations must be thread-safe. +public interface StatsStoreProvider { + + /// Returns the identifier operators use to select this store. Must be unique across all providers + /// on the classpath and stable across releases, since it appears in configuration. + String getName(); + + /// Creates a store; the caller initializes it via [StatsStore#init()]. + /// + /// @param properties statistics-related broker configuration, with keys stripped of the + /// `pinot.broker.stats.` prefix (for example `dir`) + /// @throws StatsStoreException if the store cannot be created from this configuration + StatsStore create(Map<String, String> properties) + throws StatsStoreException; +} diff --git a/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/TableStatistics.java b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/TableStatistics.java new file mode 100644 index 00000000000..f9a35fe7374 --- /dev/null +++ b/pinot-query-planner-spi/src/main/java/org/apache/pinot/query/planner/spi/stats/TableStatistics.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.spi.stats; + + +/// Immutable aggregate statistics for a table, used by the cost-based query planner. +/// +/// Instances are created via [#builder()]. Unknown numeric fields are represented +/// by `-1`; unknown timestamp fields by `0`. +/// +/// Thread-safety: immutable; safe for concurrent access. +public class TableStatistics { + private final long _rowCount; + private final StatConfidence _rowCountConfidence; + private final long _tableSizeBytes; + private final StatConfidence _sizeConfidence; + private final long _updatedAtMs; + + private TableStatistics(Builder builder) { + _rowCount = builder._rowCount; + _rowCountConfidence = builder._rowCountConfidence; + _tableSizeBytes = builder._tableSizeBytes; + _sizeConfidence = builder._sizeConfidence; + _updatedAtMs = builder._updatedAtMs; + } + + /// Returns a new [Builder] for constructing [TableStatistics] instances. + public static Builder builder() { + return new Builder(); + } + + /// Returns the estimated number of rows in the table, or `-1` if unknown. + public long getRowCount() { + return _rowCount; + } + + /// Returns the confidence level of the [#getRowCount()] value. + public StatConfidence getRowCountConfidence() { + return _rowCountConfidence; + } + + /// Returns the estimated total size of the table in bytes, or `-1` if unknown. + public long getTableSizeBytes() { + return _tableSizeBytes; + } + + /// Returns the confidence level of the [#getTableSizeBytes()] value. + public StatConfidence getSizeConfidence() { + return _sizeConfidence; + } + + /// Returns the epoch-millisecond timestamp at which these statistics were last updated, + /// or `0` if unknown. + public long getUpdatedAtMs() { + return _updatedAtMs; + } + + /// Builder for [TableStatistics]. + /// + /// Default values: numeric fields default to `-1` (unknown), + /// confidence fields default to [StatConfidence#UNKNOWN], + /// timestamp fields default to `0` (unknown). + /// + /// Thread-safety: not thread-safe; use from a single thread. + public static class Builder { + private long _rowCount = -1; + private StatConfidence _rowCountConfidence = StatConfidence.UNKNOWN; + private long _tableSizeBytes = -1; + private StatConfidence _sizeConfidence = StatConfidence.UNKNOWN; + private long _updatedAtMs = 0; + + private Builder() { + } + + /// Sets the row count and its confidence level. + public Builder rowCount(long rowCount, StatConfidence confidence) { + _rowCount = rowCount; + _rowCountConfidence = confidence; + return this; + } + + /// Sets the table size in bytes and its confidence level. + public Builder tableSizeBytes(long tableSizeBytes, StatConfidence confidence) { + _tableSizeBytes = tableSizeBytes; + _sizeConfidence = confidence; + return this; + } + + /// Sets the epoch-millisecond timestamp of the last statistics update. + public Builder updatedAtMs(long updatedAtMs) { + _updatedAtMs = updatedAtMs; + return this; + } + + /// Builds the immutable [TableStatistics] instance. + public TableStatistics build() { + return new TableStatistics(this); + } + } +} diff --git a/pinot-query-planner-spi/src/test/java/org/apache/pinot/query/planner/spi/stats/ColumnValueTypeTest.java b/pinot-query-planner-spi/src/test/java/org/apache/pinot/query/planner/spi/stats/ColumnValueTypeTest.java new file mode 100644 index 00000000000..bf4361b1dfb --- /dev/null +++ b/pinot-query-planner-spi/src/test/java/org/apache/pinot/query/planner/spi/stats/ColumnValueTypeTest.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.spi.stats; + +import java.math.BigDecimal; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; + + +/// Covers the ordering dispatch and its degradation paths. Both matter on the query-planning path: +/// a throw here would fail a query rather than cost it an estimate. +public class ColumnValueTypeTest { + + @DataProvider(name = "orderedPairs") + public Object[][] orderedPairs() { + // Each row is a value that must order BELOW the next, under that type's own ordering. + return new Object[][]{ + {ColumnValueType.LONG, "9", "10"}, + // Beyond 2^53, where a double round-trip would lose the distinction entirely. + {ColumnValueType.LONG, "9007199254740993", "9007199254740994"}, + {ColumnValueType.DOUBLE, "1.5", "10.5"}, + {ColumnValueType.DOUBLE, "-2.5", "-1.5"}, + {ColumnValueType.BIG_DECIMAL, "9.10", "10.01"}, + // Lexical, which is exactly where a numeric ordering would disagree. + {ColumnValueType.STRING, "10", "9"}, + }; + } + + @Test(dataProvider = "orderedPairs") + public void testCompareOrdersByTheDeclaredType(ColumnValueType type, String lower, String higher) { + assertTrue(type.compare(lower, higher) < 0, type + ": " + lower + " should order below " + higher); + assertTrue(type.compare(higher, lower) > 0, type + ": " + higher + " should order above " + lower); + assertEquals(type.compare(lower, lower), 0); + } + + @DataProvider(name = "types") + public Object[][] types() { + return new Object[][]{ + {ColumnValueType.LONG}, {ColumnValueType.DOUBLE}, {ColumnValueType.BIG_DECIMAL}, {ColumnValueType.STRING} + }; + } + + @Test(dataProvider = "types") + public void testCompareFallsBackToLexicalOnMalformedValues(ColumnValueType type) { + // A stored value that does not parse must degrade, not throw. + assertEquals(type.compare("abc", "abd"), "abc".compareTo("abd")); + assertEquals(type.compare("abc", "abc"), 0); + // One side malformed is still enough to force the lexical path. + assertEquals(type.compare("1", "abc"), "1".compareTo("abc")); + } + + @Test + public void testToComparableDeserializesByType() { + assertEquals(ColumnValueType.LONG.toComparable("42"), 42L); + assertEquals(ColumnValueType.DOUBLE.toComparable("42.5"), 42.5d); + assertEquals(ColumnValueType.BIG_DECIMAL.toComparable("42.50"), new BigDecimal("42.50")); + assertEquals(ColumnValueType.STRING.toComparable("42"), "42"); + } + + @Test(dataProvider = "types") + public void testToComparableReturnsRawValueWhenUnparseable(ColumnValueType type) { + assertEquals(type.toComparable("not-a-number"), "not-a-number"); + } + + @Test(dataProvider = "types") + public void testToComparableKeepsNull(ColumnValueType type) { + assertNull(type.toComparable(null)); + } + + @Test + public void testFromNameResolvesEveryConstant() { + for (ColumnValueType type : ColumnValueType.values()) { + assertEquals(ColumnValueType.fromName(type.name()), type); + } + } + + @Test + public void testFromNameYieldsNullForUnknownNames() { + // A store written by a newer build may carry a name this one does not have. That must cost + // precision (null means "ordering unknown"), not throw out of the planning path. + assertNull(ColumnValueType.fromName("TIMESTAMP_WITH_TIMEZONE")); + assertNull(ColumnValueType.fromName("")); + assertNull(ColumnValueType.fromName("long")); + assertNull(ColumnValueType.fromName(null)); + } +} diff --git a/pinot-query-planner-spi/src/test/java/org/apache/pinot/query/planner/spi/stats/StatsAggregationsTest.java b/pinot-query-planner-spi/src/test/java/org/apache/pinot/query/planner/spi/stats/StatsAggregationsTest.java new file mode 100644 index 00000000000..128df01a89a --- /dev/null +++ b/pinot-query-planner-spi/src/test/java/org/apache/pinot/query/planner/spi/stats/StatsAggregationsTest.java @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.spi.stats; + +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; + + +/// Covers the rollup semantics both stores fold through, so they cannot drift apart. +public class StatsAggregationsTest { + + private static final String COLUMN = "col"; + + private static SegmentColumnStatsRow row(String segment, long ndv, String min, String max, boolean minTrusted, + double avgBytes, double nullFraction, ColumnValueType valueType) { + return new SegmentColumnStatsRow(segment, COLUMN, ndv, min, max, minTrusted, avgBytes, nullFraction, valueType); + } + + @Test + public void testEmptyAccumulatorReportsNoStatistics() { + assertTrue(new StatsAggregations.ColumnStatsAccumulator().isEmpty()); + } + + @Test + public void testBoundsUseTheRecordedOrdering() { + StatsAggregations.ColumnStatsAccumulator acc = new StatsAggregations.ColumnStatsAccumulator(); + acc.add(100, row("s1", 5, "9", "9", true, 4, 0.0, ColumnValueType.LONG)); + acc.add(100, row("s2", 7, "10", "10", true, 4, 0.0, ColumnValueType.LONG)); + ColumnStatistics stats = acc.build(COLUMN); + // Lexically "10" < "9"; numerically it is not. The recorded type decides. + assertEquals(stats.getMinValue(), 9L); + assertEquals(stats.getMaxValue(), 10L); + assertTrue(stats.isMinTrusted()); + } + + @Test + public void testStringColumnKeepsLexicalOrdering() { + StatsAggregations.ColumnStatsAccumulator acc = new StatsAggregations.ColumnStatsAccumulator(); + acc.add(100, row("s1", 5, "9", "9", true, 4, 0.0, ColumnValueType.STRING)); + acc.add(100, row("s2", 7, "10", "10", true, 4, 0.0, ColumnValueType.STRING)); + ColumnStatistics stats = acc.build(COLUMN); + assertEquals(stats.getMinValue(), "10"); + assertEquals(stats.getMaxValue(), "9"); + } + + @Test + public void testUnrecordedTypeDropsBothBounds() { + StatsAggregations.ColumnStatsAccumulator acc = new StatsAggregations.ColumnStatsAccumulator(); + acc.add(100, row("s1", 5, "1", "500", true, 4, 0.0, null)); + ColumnStatistics stats = acc.build(COLUMN); + // Bounds folded without a known ordering are neither a true minimum nor a true maximum, and + // isMinTrusted says nothing about the maximum -- so neither is reported at all. + assertNull(stats.getMinValue()); + assertNull(stats.getMaxValue()); + assertFalse(stats.isMinTrusted()); + // NDV is unaffected: it needs no ordering. + assertEquals(stats.getNdv(), 5); + } + + @Test + public void testConflictingTypesAcrossSegmentsDropBothBounds() { + StatsAggregations.ColumnStatsAccumulator acc = new StatsAggregations.ColumnStatsAccumulator(); + acc.add(100, row("s1", 5, "1", "500", true, 4, 0.0, ColumnValueType.LONG)); + acc.add(100, row("s2", 5, "a", "z", true, 4, 0.0, ColumnValueType.STRING)); + ColumnStatistics stats = acc.build(COLUMN); + assertNull(stats.getMinValue()); + assertNull(stats.getMaxValue()); + assertFalse(stats.isMinTrusted()); + } + + @Test + public void testUntrustedMinInAnySegmentTaintsTheAggregate() { + StatsAggregations.ColumnStatsAccumulator acc = new StatsAggregations.ColumnStatsAccumulator(); + acc.add(100, row("s1", 5, "1", "5", true, 4, 0.0, ColumnValueType.LONG)); + acc.add(100, row("s2", 5, "2", "6", false, 4, 0.0, ColumnValueType.LONG)); + assertFalse(acc.build(COLUMN).isMinTrusted()); + } + + @Test + public void testWeightedAveragesAreDocumentWeighted() { + StatsAggregations.ColumnStatsAccumulator acc = new StatsAggregations.ColumnStatsAccumulator(); + acc.add(300, row("s1", 5, "1", "5", true, 8, 0.4, ColumnValueType.LONG)); + acc.add(100, row("s2", 5, "1", "5", true, 4, 0.0, ColumnValueType.LONG)); + ColumnStatistics stats = acc.build(COLUMN); + assertEquals(stats.getAvgBytesPerValue(), (8 * 300 + 4 * 100) / 400.0, 1e-9); + assertEquals(stats.getNullFraction(), (0.4 * 300) / 400.0, 1e-9); + } + + @Test + public void testUnknownSentinelsAreExcludedFromAverages() { + StatsAggregations.ColumnStatsAccumulator acc = new StatsAggregations.ColumnStatsAccumulator(); + acc.add(1000, row("s1", 5, "1", "5", true, -1, -1, ColumnValueType.LONG)); + acc.add(1000, row("s2", 5, "1", "5", true, 4, 0.2, ColumnValueType.LONG)); + ColumnStatistics stats = acc.build(COLUMN); + // Weighting the -1 sentinel in as a measurement would yield 1.5 bytes and a NEGATIVE null + // fraction -- neither a legal value nor the sentinel a consumer tests for. + assertEquals(stats.getAvgBytesPerValue(), 4.0, 1e-9); + assertEquals(stats.getNullFraction(), 0.2, 1e-9); + } + + @Test + public void testAveragesReportUnknownWhenEverySegmentIsUnknown() { + StatsAggregations.ColumnStatsAccumulator acc = new StatsAggregations.ColumnStatsAccumulator(); + acc.add(1000, row("s1", 5, "1", "5", true, -1, -1, ColumnValueType.LONG)); + ColumnStatistics stats = acc.build(COLUMN); + assertEquals(stats.getAvgBytesPerValue(), -1.0, 1e-9); + assertEquals(stats.getNullFraction(), -1.0, 1e-9); + } + + @Test + public void testNdvTakesTheLargestSegmentValue() { + StatsAggregations.ColumnStatsAccumulator acc = new StatsAggregations.ColumnStatsAccumulator(); + acc.add(100, row("s1", -1, "1", "5", true, 4, 0.0, ColumnValueType.LONG)); + acc.add(100, row("s2", 12, "1", "5", true, 4, 0.0, ColumnValueType.LONG)); + // MAX over segments is a lower bound on the table-wide NDV; the -1 sentinel cannot win it. + assertEquals(acc.build(COLUMN).getNdv(), 12); + } + + @Test + public void testOverlapRowsInterpolatesPartialOverlap() { + // Fully contained. + assertEquals(StatsAggregations.overlapRows(100, 10, 20, 0, 100), 100); + // No overlap at all. + assertEquals(StatsAggregations.overlapRows(100, 10, 20, 30, 40), 0); + // Half the segment's span falls in the range, so half its rows are attributed to it. + assertEquals(StatsAggregations.overlapRows(100, 0, 100, 0, 50), 50); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
