This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7c330d2c3611d5125a2ea838eab82fc58e7ab0e7
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Oct 15 09:30:27 2026 +0800
branch-4.1: [fix](be) Preserve the sample triggering QuantileState digest
conversion #69037 (#69049)
Cherry-picked from #69037
---------
Co-authored-by: TengJianPing <[email protected]>
---
be/src/core/value/quantile_state.cpp | 2 +-
be/src/util/tdigest.h | 3 +-
be/test/core/value/quantile_state_test.cpp | 60 +++++++++++++++++++
be/test/util/tdigest_test.cpp | 29 +++++++++
.../test_aggregate_all_functions.out | 2 +-
.../test_aggregate_all_functions.out | 2 +-
.../test_quantile_state_explicit_boundary.out | 22 +++++++
.../test_quantile_state_explicit_boundary.groovy | 70 ++++++++++++++++++++++
8 files changed, 186 insertions(+), 4 deletions(-)
diff --git a/be/src/core/value/quantile_state.cpp
b/be/src/core/value/quantile_state.cpp
index 9ca0f0a0ca8..47df852e2ad 100644
--- a/be/src/core/value/quantile_state.cpp
+++ b/be/src/core/value/quantile_state.cpp
@@ -411,10 +411,10 @@ void QuantileState::add_value(const double& value) {
for (int i = 0; i < _explicit_data.size(); i++) {
_tdigest_ptr->digest.add((float)_explicit_data[i]);
}
+ _tdigest_ptr->digest.add((float)value);
_explicit_data.clear();
_explicit_data.shrink_to_fit();
_type = TDIGEST;
-
} else {
_explicit_data.emplace_back(value);
}
diff --git a/be/src/util/tdigest.h b/be/src/util/tdigest.h
index 158dc748ce9..a37a606db01 100644
--- a/be/src/util/tdigest.h
+++ b/be/src/util/tdigest.h
@@ -544,7 +544,8 @@ private:
Value _min = std::numeric_limits<Value>::max();
- Value _max = std::numeric_limits<Value>::min();
+ // min() is the smallest positive value, so use lowest() for non-positive
inputs.
+ Value _max = std::numeric_limits<Value>::lowest();
Index _max_processed;
diff --git a/be/test/core/value/quantile_state_test.cpp
b/be/test/core/value/quantile_state_test.cpp
index c025294caa3..e557fe2ce65 100644
--- a/be/test/core/value/quantile_state_test.cpp
+++ b/be/test/core/value/quantile_state_test.cpp
@@ -149,6 +149,66 @@ static long serialized_digest_weight(const QuantileState&
state) {
return digest.totalWeight();
}
+static void expect_digest_samples(const QuantileState& state, int count,
double minimum,
+ double maximum) {
+ ASSERT_EQ(TDIGEST, state._type);
+ EXPECT_EQ(count, serialized_digest_weight(state));
+ EXPECT_EQ(minimum, state.get_value_by_percentile(0));
+ EXPECT_EQ(maximum, state.get_value_by_percentile(1));
+}
+
+TEST(QuantileStateTest, AddValueRetainsSampleAtExplicitLimit) {
+ for (float compression : {QUANTILE_STATE_COMPRESSION_MIN,
QUANTILE_STATE_COMPRESSION_MAX}) {
+ for (double value : {-100.0, 100.0}) {
+ QuantileState state(compression);
+ for (int i = 0; i < QUANTILE_STATE_EXPLICIT_NUM - 1; ++i) {
+ state.add_value(0);
+ }
+ ASSERT_EQ(EXPLICIT, state._type);
+ EXPECT_EQ(QUANTILE_STATE_EXPLICIT_NUM - 1,
state._explicit_data.size());
+ state.add_value(0);
+ ASSERT_EQ(EXPLICIT, state._type);
+ EXPECT_EQ(QUANTILE_STATE_EXPLICIT_NUM,
state._explicit_data.size());
+
+ state.add_value(value);
+ EXPECT_TRUE(state._explicit_data.empty());
+ expect_digest_samples(state, QUANTILE_STATE_EXPLICIT_NUM + 1,
std::min(0.0, value),
+ std::max(0.0, value));
+
+ state.add_value(0);
+ expect_digest_samples(state, QUANTILE_STATE_EXPLICIT_NUM + 2,
std::min(0.0, value),
+ std::max(0.0, value));
+ }
+ }
+}
+
+TEST(QuantileStateTest, MergeSingletonAndExplicitRetainsBoundarySample) {
+ for (double value : {-100.0, 100.0}) {
+ auto explicit_state = constant_state(0, QUANTILE_STATE_EXPLICIT_NUM);
+ auto singleton = constant_state(value, 1);
+
+ auto explicit_first = explicit_state;
+ explicit_first.merge(singleton);
+ auto singleton_first = singleton;
+ singleton_first.merge(explicit_state);
+ expect_digest_samples(explicit_first, QUANTILE_STATE_EXPLICIT_NUM + 1,
std::min(0.0, value),
+ std::max(0.0, value));
+ expect_digest_samples(singleton_first, QUANTILE_STATE_EXPLICIT_NUM + 1,
+ std::min(0.0, value), std::max(0.0, value));
+ EXPECT_EQ(QUANTILE_STATE_EXPLICIT_NUM,
explicit_state._explicit_data.size());
+ EXPECT_EQ(0, explicit_state.get_value_by_percentile(1));
+ EXPECT_EQ(value, singleton.get_value_by_percentile(1));
+ }
+}
+
+TEST(QuantileStateTest, ExplicitMergeGroupingRetainsAllSamples) {
+ auto state = constant_state(0, QUANTILE_STATE_EXPLICIT_NUM - 1);
+ auto other = constant_state(0, 1);
+ other.add_value(100);
+ state.merge(other);
+ expect_digest_samples(state, QUANTILE_STATE_EXPLICIT_NUM + 1, 0, 100);
+}
+
TEST(QuantileStateTest, SelfMergeAndSharedSourceMerge) {
for (int count : {2, 4096}) {
auto state = constant_state(10, count);
diff --git a/be/test/util/tdigest_test.cpp b/be/test/util/tdigest_test.cpp
index 0804fa3ad7f..8d7c0408f5a 100644
--- a/be/test/util/tdigest_test.cpp
+++ b/be/test/util/tdigest_test.cpp
@@ -123,6 +123,35 @@ TEST_F(TDigestTest, EmptyDigest) {
EXPECT_EQ(0, digest.processed().size());
}
+TEST_F(TDigestTest, NonPositiveValuesHaveCorrectMaximum) {
+ for (const auto& values : {std::vector<Value> {-3, -2, -1},
std::vector<Value> {-3, -2, 0}}) {
+ SCOPED_TRACE(values.back());
+ const auto expect_extrema = [&](TDigest& digest) {
+ EXPECT_EQ(values.front(), digest.quantile(0));
+ EXPECT_EQ(values.back(), digest.quantile(1));
+ };
+ TDigest digest(2048);
+ for (auto value : values) {
+ digest.add(value);
+ }
+ expect_extrema(digest);
+
+ std::vector<uint8_t> bytes(digest.serialized_size());
+ ASSERT_EQ(bytes.size(), digest.serialize(bytes.data()));
+ TDigest restored(0);
+ restored.unserialize(bytes.data());
+ expect_extrema(restored);
+
+ TDigest merged(2048);
+ merged.merge(&digest);
+ expect_extrema(merged);
+
+ std::vector<Centroid> processed(digest.processed());
+ TDigest from_centroids(std::move(processed), {}, 2048, 0, 0);
+ expect_extrema(from_centroids);
+ }
+}
+
TEST_F(TDigestTest, SingleValue) {
TDigest digest(100);
std::random_device gen;
diff --git
a/regression-test/data/nereids_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.out
b/regression-test/data/nereids_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.out
index ac05bf4e2aa..45cdeea68ce 100644
---
a/regression-test/data/nereids_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.out
+++
b/regression-test/data/nereids_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.out
@@ -267,7 +267,7 @@ beijing chengdu shanghai
-- !select49 --
20220201 0 1.0
20220201 1 1.0
-20220202 2 2500.0
+20220202 2 2499.5
-- !select50 --
20220201 0 1.0
diff --git
a/regression-test/data/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.out
b/regression-test/data/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.out
index 55298776e38..8d7b0042dee 100644
---
a/regression-test/data/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.out
+++
b/regression-test/data/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.out
@@ -303,7 +303,7 @@ beijing chengdu shanghai
-- !select49 --
20220201 0 1.0
20220201 1 1.0
-20220202 2 2500.0
+20220202 2 2499.5
-- !select50 --
20220201 0 1.0
diff --git
a/regression-test/data/query_p0/sql_functions/quantile_state_functions/test_quantile_state_explicit_boundary.out
b/regression-test/data/query_p0/sql_functions/quantile_state_functions/test_quantile_state_explicit_boundary.out
new file mode 100644
index 00000000000..12546dfd2f2
--- /dev/null
+++
b/regression-test/data/query_p0/sql_functions/quantile_state_functions/test_quantile_state_explicit_boundary.out
@@ -0,0 +1,22 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !partial_extrema --
+0 0 0 0
+0 1 100 100
+1 0 100 100
+1 1 0 0
+2 0 0 0
+2 1 0 100
+
+-- !merged_extrema --
+0 0 100
+1 0 100
+2 0 100
+
+-- !ordered_merge_extrema --
+0 0 0 0
+0 1 0 100
+1 0 100 100
+1 1 0 100
+2 0 0 0
+2 1 0 100
+
diff --git
a/regression-test/suites/query_p0/sql_functions/quantile_state_functions/test_quantile_state_explicit_boundary.groovy
b/regression-test/suites/query_p0/sql_functions/quantile_state_functions/test_quantile_state_explicit_boundary.groovy
new file mode 100644
index 00000000000..94fae7ff8fe
--- /dev/null
+++
b/regression-test/suites/query_p0/sql_functions/quantile_state_functions/test_quantile_state_explicit_boundary.groovy
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_quantile_state_explicit_boundary") {
+ sql "DROP TABLE IF EXISTS test_quantile_state_boundary_partials"
+ sql """
+ CREATE TABLE test_quantile_state_boundary_partials (
+ layout_id INT NOT NULL,
+ part INT NOT NULL,
+ s QUANTILE_STATE QUANTILE_UNION NOT NULL
+ ) AGGREGATE KEY(layout_id, part)
+ DISTRIBUTED BY HASH(layout_id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ // The same 2048 zeros and singleton 100 are grouped three ways:
+ // EXPLICIT + SINGLE, SINGLE + EXPLICIT, and EXPLICIT + EXPLICIT.
+ sql """
+ INSERT INTO test_quantile_state_boundary_partials
+ SELECT 0, CAST(number = 2048 AS INT),
+ quantile_union(to_quantile_state(IF(number = 2048, 100.0, 0.0),
2048))
+ FROM numbers("number" = "2049") GROUP BY 2
+ """
+ sql """
+ INSERT INTO test_quantile_state_boundary_partials
+ SELECT 1, CAST(number != 2048 AS INT),
+ quantile_union(to_quantile_state(IF(number = 2048, 100.0, 0.0),
2048))
+ FROM numbers("number" = "2049") GROUP BY 2
+ """
+ sql """
+ INSERT INTO test_quantile_state_boundary_partials
+ SELECT 2, CAST(number >= 2047 AS INT),
+ quantile_union(to_quantile_state(IF(number = 2048, 100.0, 0.0),
2048))
+ FROM numbers("number" = "2049") GROUP BY 2
+ """
+
+ qt_partial_extrema """
+ SELECT layout_id, part, quantile_percent(s, 0), quantile_percent(s, 1)
+ FROM test_quantile_state_boundary_partials ORDER BY layout_id, part
+ """
+ qt_merged_extrema """
+ SELECT layout_id, quantile_percent(quantile_union(s), 0),
+ quantile_percent(quantile_union(s), 1)
+ FROM test_quantile_state_boundary_partials GROUP BY layout_id ORDER BY
layout_id
+ """
+ // Window ordering guarantees both directions of the boundary conversion.
+ qt_ordered_merge_extrema """
+ SELECT layout_id, part, quantile_percent(s, 0), quantile_percent(s, 1)
+ FROM (
+ SELECT layout_id, part, quantile_union(s) OVER (
+ PARTITION BY layout_id ORDER BY part
+ ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS s
+ FROM test_quantile_state_boundary_partials
+ ) t ORDER BY layout_id, part
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
