This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 01e735b29c8 [fix](be) Fix condition cache and COUNT_ON_INDEX predicate
handling (#69087)
01e735b29c8 is described below
commit 01e735b29c8b8803b9e2df10b46b9b02db1e6251
Author: Chenyang Sun <[email protected]>
AuthorDate: Mon Oct 19 10:19:04 2026 +0800
[fix](be) Fix condition cache and COUNT_ON_INDEX predicate handling (#69087)
### What problem does this PR solve?
### 1. Segment TopN leaves an incomplete condition cache
SQL:
```sql
-- c2 is a DUPLICATE KEY table with multiple overlapping segments.
SELECT k, v FROM c2 WHERE abs(v) > 0 ORDER BY k LIMIT 1;
SELECT count(*), sum(k) FROM c2 WHERE abs(v) > 0;
SELECT k, v FROM c2 WHERE abs(v) > 1 ORDER BY k DESC LIMIT 1;
SELECT count(*), sum(k) FROM c2 WHERE abs(v) > 1;
Before this fix:
- abs(v) > 0 returns 7022 rows instead of 14043.
- abs(v) > 1 returns 4681 rows instead of 9362.
The segment reaches its local read limit and writes an incomplete
condition-cache entry. A later query reuses the entry and skips valid row
ranges.
After this fix:
- abs(v) > 0 returns 14043 rows.
- abs(v) > 1 returns 9362 rows.
A segment that reaches its local read limit no longer writes the incomplete
condition-cache entry.
### 2. COUNT_ON_INDEX rejects valid slotless predicates
SQL:
-- t8 is a DUPLICATE KEY table with an inverted index on v.
SELECT count(*) FROM t8 WHERE random() < 2;
SELECT count(*) FROM t8 WHERE v MATCH_ANY 'apple' AND random() < 2;
Before this fix:
Both queries fail with:
INVALID_ARGUMENT:
COUNT_ON_INDEX pushdown cannot be used with residual scan predicates
After this fix:
- The first query returns 3.
- The second query returns 2.
COUNT_ON_INDEX now rejects only residual predicates that reference columns.
Slotless predicates are evaluated by the scanner.
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR should
merge into -->
---
be/src/exec/operator/olap_scan_operator.cpp | 18 +++--
be/src/storage/segment/segment_iterator.cpp | 3 +-
.../test_segment_limit_condition_cache.out | 13 ++++
.../test_count_on_index_slotless_predicate.out | 13 ++++
.../test_segment_limit_condition_cache.groovy | 76 ++++++++++++++++++++++
.../test_count_on_index_slotless_predicate.groovy | 49 ++++++++++++++
6 files changed, 165 insertions(+), 7 deletions(-)
diff --git a/be/src/exec/operator/olap_scan_operator.cpp
b/be/src/exec/operator/olap_scan_operator.cpp
index 2c8a0514bd1..d0b4f4150cf 100644
--- a/be/src/exec/operator/olap_scan_operator.cpp
+++ b/be/src/exec/operator/olap_scan_operator.cpp
@@ -23,6 +23,7 @@
#include <memory>
#include <numeric>
#include <optional>
+#include <set>
#include <shared_mutex>
#include "cloud/cloud_meta_mgr.h"
@@ -499,13 +500,18 @@ static Status
validate_residual_scan_conjuncts(RuntimeState* state,
}
}
- if (push_down_agg_type == TPushAggOp::COUNT_ON_INDEX &&
!conjuncts.empty()) {
+ // Reject only residual predicates that reference columns: COUNT_ON_INDEX
fills columns with
+ // defaults, while a predicate such as random() < 2 still filters each row
correctly.
+ if (push_down_agg_type == TPushAggOp::COUNT_ON_INDEX &&
+ std::ranges::any_of(conjuncts, [](const VExprContextSPtr& conjunct) {
+ std::set<int> column_ids;
+ conjunct->root()->collect_slot_column_ids(column_ids);
+ return !column_ids.empty();
+ })) {
return Status::InvalidArgument(
- "COUNT_ON_INDEX pushdown cannot be used with residual scan
predicates. "
- "Residual predicates must be evaluated before COUNT_ON_INDEX
counts rows; "
- "otherwise the query may return incorrect results. "
- "enable_segment_limit_pushdown only controls SegmentIterator
LIMIT pushdown and "
- "does not make COUNT_ON_INDEX safe with residual predicates.
Set "
+ "COUNT_ON_INDEX pushdown cannot be used with residual scan
predicates that "
+ "reference columns. Residual predicates must be evaluated
before COUNT_ON_INDEX "
+ "counts rows; otherwise the query may return incorrect
results. Set "
"enable_count_on_index_pushdown=false to disable
COUNT_ON_INDEX pushdown.");
}
return Status::OK();
diff --git a/be/src/storage/segment/segment_iterator.cpp
b/be/src/storage/segment/segment_iterator.cpp
index 807351e94d1..e2ec36a0804 100644
--- a/be/src/storage/segment/segment_iterator.cpp
+++ b/be/src/storage/segment/segment_iterator.cpp
@@ -2892,7 +2892,8 @@ Status SegmentIterator::next_batch(Block* block) {
block->replace_by_position(cid, type->create_column());
}
- if (_opts.condition_cache_digest && !_find_condition_cache) {
+ if (_opts.condition_cache_digest && !_find_condition_cache &&
+ !(_opts.read_limit > 0 && _rows_returned >=
_opts.read_limit)) {
auto* condition_cache = ConditionCache::instance();
ConditionCache::CacheKey cache_key(_opts.rowset_id,
_segment->id(),
_opts.condition_cache_digest);
diff --git
a/regression-test/data/fault_injection_p0/test_segment_limit_condition_cache.out
b/regression-test/data/fault_injection_p0/test_segment_limit_condition_cache.out
new file mode 100644
index 00000000000..ca492eb51b6
--- /dev/null
+++
b/regression-test/data/fault_injection_p0/test_segment_limit_condition_cache.out
@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !topn_asc --
+0 -3
+
+-- !all_after_asc --
+14043 115029723
+
+-- !topn_desc --
+16381 -2
+
+-- !all_after_desc --
+9362 76686481
+
diff --git
a/regression-test/data/inverted_index_p0/test_count_on_index_slotless_predicate.out
b/regression-test/data/inverted_index_p0/test_count_on_index_slotless_predicate.out
new file mode 100644
index 00000000000..e6b76de295b
--- /dev/null
+++
b/regression-test/data/inverted_index_p0/test_count_on_index_slotless_predicate.out
@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !all_rows --
+3
+
+-- !no_rows --
+0
+
+-- !match_all_rows --
+2
+
+-- !match_no_rows --
+0
+
diff --git
a/regression-test/suites/fault_injection_p0/test_segment_limit_condition_cache.groovy
b/regression-test/suites/fault_injection_p0/test_segment_limit_condition_cache.groovy
new file mode 100644
index 00000000000..a1a2b95b8ce
--- /dev/null
+++
b/regression-test/suites/fault_injection_p0/test_segment_limit_condition_cache.groovy
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A storage TopN (ORDER BY key LIMIT n) stops each segment after n rows. It
must not leave
+// a condition cache entry behind, or a later query with the same predicate
skips row ranges
+// that do have matching rows.
+//
+// The bug needs a rowset with overlapping segments, which is read by
VMergeIterator: it asks
+// every segment for one more batch, so a segment that hit its LIMIT reaches
EOF.
+// MemTable.need_flush writes one segment per inserted block.
+suite("test_segment_limit_condition_cache", "nonConcurrent") {
+ def tbl = "test_segment_limit_condition_cache"
+ sql "DROP TABLE IF EXISTS ${tbl}"
+ sql """
+ CREATE TABLE ${tbl} (k INT, v INT)
+ DUPLICATE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES ('replication_num' = '1', 'disable_auto_compaction' =
'true')
+ """
+
+ try {
+ GetDebugPoint().enableDebugPointForAllBEs("MemTable.need_flush")
+ // 4 blocks of 4096 rows -> 4 segments, each larger than one 2048-row
cache granule.
+ // Every block has keys from the whole range, so the segments overlap.
+ sql "SET batch_size = 4096"
+ sql """
+ INSERT INTO ${tbl}
+ SELECT (number * 7919) % 16384, (number * 7919) % 16384 % 7 - 3
+ FROM numbers('number' = '16384')
+ """
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs("MemTable.need_flush")
+ }
+
+ def tablets = sql_return_maparray("SHOW TABLETS FROM ${tbl}")
+ def (code, out, err) = curl("GET", tablets[0].MetaUrl)
+ assertEquals(0, code)
+ def hasMergeLayout = parseJson(out.trim()).rs_metas.any { meta ->
+ meta.segments_overlap_pb == "OVERLAPPING" && meta.num_segments > 1
+ }
+ assertTrue(hasMergeLayout, "expected an OVERLAPPING rowset with more than
one segment")
+
+ // Both switches are randomized in fuzzy runs; this case needs both on.
+ sql "SET enable_condition_cache = true"
+ sql "SET enable_segment_limit_pushdown = true"
+ // Disable TopN runtime filters so the queries exercise storage TopN and
condition cache.
+ sql "SET topn_filter_ratio = 0"
+ // Keep each segment on the merge path that requests EOF after its local
LIMIT.
+ sql "SET enable_parallel_scan = false"
+ explain {
+ sql "SELECT k, v FROM ${tbl} WHERE abs(v) > 0 ORDER BY k LIMIT 1"
+ contains "SORT LIMIT: 1"
+ notContains "TOPN filter targets"
+ }
+
+ // Each case runs the TopN first, then the same predicate without LIMIT.
+ qt_topn_asc "SELECT k, v FROM ${tbl} WHERE abs(v) > 0 ORDER BY k LIMIT 1"
+ qt_all_after_asc "SELECT count(*), sum(k) FROM ${tbl} WHERE abs(v) > 0"
+
+ qt_topn_desc "SELECT k, v FROM ${tbl} WHERE abs(v) > 1 ORDER BY k DESC
LIMIT 1"
+ qt_all_after_desc "SELECT count(*), sum(k) FROM ${tbl} WHERE abs(v) > 1"
+}
diff --git
a/regression-test/suites/inverted_index_p0/test_count_on_index_slotless_predicate.groovy
b/regression-test/suites/inverted_index_p0/test_count_on_index_slotless_predicate.groovy
new file mode 100644
index 00000000000..a5920f81488
--- /dev/null
+++
b/regression-test/suites/inverted_index_p0/test_count_on_index_slotless_predicate.groovy
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A predicate that references no column, such as random() < 2, is not pushed
into the storage
+// layer and stays in the scanner. COUNT_ON_INDEX must still count with it
instead of failing.
+suite("test_count_on_index_slotless_predicate", "p0") {
+ def tbl = "test_count_on_index_slotless_predicate"
+ sql "DROP TABLE IF EXISTS ${tbl}"
+ sql """
+ CREATE TABLE ${tbl} (
+ k INT,
+ v STRING,
+ INDEX idx_v (v) USING INVERTED PROPERTIES ('parser' = 'english')
+ ) DUPLICATE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES ('replication_num' = '1')
+ """
+ sql "INSERT INTO ${tbl} VALUES (1, 'apple pie'), (2, 'banana'), (3, 'apple
juice')"
+
+ sql "SET enable_count_on_index_pushdown = true"
+ explain {
+ sql "SELECT count(*) FROM ${tbl} WHERE random() < 2"
+ contains "pushAggOp=COUNT_ON_INDEX"
+ }
+ explain {
+ sql "SELECT count(*) FROM ${tbl} WHERE v MATCH_ANY 'apple' AND
random() < 2"
+ contains "pushAggOp=COUNT_ON_INDEX"
+ }
+
+ // random() returns a value in [0, 1), so "< 2" keeps every row and "< 0"
keeps none.
+ qt_all_rows "SELECT count(*) FROM ${tbl} WHERE random() < 2"
+ qt_no_rows "SELECT count(*) FROM ${tbl} WHERE random() < 0"
+ qt_match_all_rows "SELECT count(*) FROM ${tbl} WHERE v MATCH_ANY 'apple'
AND random() < 2"
+ qt_match_no_rows "SELECT count(*) FROM ${tbl} WHERE v MATCH_ANY 'apple'
AND random() < 0"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
