This is an automated email from the ASF dual-hosted git repository.
csun5285 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7ff8c69da8f [fix](fe) Do not use short-circuit point query when
reading hidden columns (#70162)
7ff8c69da8f is described below
commit 7ff8c69da8ffb2907a9d3a9adcbe5564fd13308d
Author: Chenyang Sun <[email protected]>
AuthorDate: Wed Nov 4 17:31:26 2026 +0800
[fix](fe) Do not use short-circuit point query when reading hidden columns
(#70162)
Problem Summary:
A singleton rowset stores only a placeholder for `__DORIS_VERSION_COL__`
and `__DORIS_COMMIT_TSO_COL__`; the real value comes from the rowset
metadata at read time. The short-circuit point query reads the JSONB row
store, which holds that placeholder, and the BE fast path deliberately
does not resolve these columns (see `point_query_executor.cpp`). So on a
MOW table with `store_row_column = true`:
```sql
CREATE TABLE b (id INT NOT NULL, v INT) UNIQUE KEY(id) DISTRIBUTED BY
HASH(id) BUCKETS 1
PROPERTIES ('replication_num'='1',
'enable_unique_key_merge_on_write'='true',
'store_row_column'='true', 'changelog.enable'='true');
INSERT INTO b VALUES (1, 7), (2, 8);
SELECT __DORIS_VERSION_COL__, __DORIS_COMMIT_TSO_COL__, v FROM b WHERE id =
1;
-- before: 0, 0, 7 after: 2, <commit tso>, 7
```
---
.../LogicalResultSinkToShortCircuitPointQuery.java | 10 +++-
.../rules/rewrite/ShortCircuitPointQueryTest.java | 53 ++++++++++++++++++
.../test_short_circuit_rowset_hidden_columns.out | 10 ++++
...test_short_circuit_rowset_hidden_columns.groovy | 65 ++++++++++++++++++++++
4 files changed, 137 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java
index e0bb0ea45b9..235f203a0cb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java
@@ -26,6 +26,7 @@ import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
@@ -95,6 +96,11 @@ public class LogicalResultSinkToShortCircuitPointQuery
implements RewriteRuleFac
&& olapTable.storeRowColumn();
}
+ // Queries reading hidden columns do not use the short circuit.
+ private boolean readsHiddenColumn(Set<Slot> slots) {
+ return slots.stream().anyMatch(slot -> !((SlotReference)
slot).isVisible());
+ }
+
// set short circuit flag and return the original plan
private Plan shortCircuit(Plan root, OlapTable olapTable,
Set<Expression> conjuncts, StatementContext statementContext) {
@@ -119,7 +125,8 @@ public class LogicalResultSinkToShortCircuitPointQuery
implements RewriteRuleFac
RuleType.SHOR_CIRCUIT_POINT_QUERY.build(
logicalResultSink(logicalProject(logicalFilter(logicalOlapScan()
.when(this::scanMatchShortCircuitCondition)
- ).when(this::filterMatchShortCircuitCondition)))
+ ).when(this::filterMatchShortCircuitCondition))
+ .when(project ->
!readsHiddenColumn(project.getInputSlots())))
.thenApply(ctx -> {
return shortCircuit(ctx.root,
ctx.root.child().child().child().getTable(),
@@ -129,6 +136,7 @@ public class LogicalResultSinkToShortCircuitPointQuery
implements RewriteRuleFac
logicalResultSink(logicalFilter(logicalOlapScan()
.when(this::scanMatchShortCircuitCondition)
).when(this::filterMatchShortCircuitCondition))
+ .when(sink ->
!readsHiddenColumn(sink.getInputSlots()))
.thenApply(ctx -> {
return shortCircuit(ctx.root,
ctx.root.child().child().getTable(),
ctx.root.child().getConjuncts(),
ctx.statementContext);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ShortCircuitPointQueryTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ShortCircuitPointQueryTest.java
index 9511e8d4bb9..5232dbfc4df 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ShortCircuitPointQueryTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ShortCircuitPointQueryTest.java
@@ -21,6 +21,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Tablet;
+import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
import org.apache.doris.datasource.doris.RemoteOlapTable;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
@@ -48,6 +49,7 @@ class ShortCircuitPointQueryTest extends TestWithFeService
@Override
protected void runBeforeAll() throws Exception {
+ Config.enable_feature_changelog = true;
createDatabase("test");
useDatabase("test");
createTable("CREATE TABLE `tbl_point_query` (\n"
@@ -78,6 +80,57 @@ class ShortCircuitPointQueryTest extends TestWithFeService
+ " \"light_schema_change\" = \"true\",\n"
+ " \"store_row_column\" = \"true\"\n"
+ ");");
+ // Carries both __DORIS_VERSION_COL__ and __DORIS_COMMIT_TSO_COL__.
+ createTable("CREATE TABLE `tbl_changelog_point_query` (\n"
+ + " `key` int(11) NULL,\n"
+ + " `v1` varchar(30) NULL\n"
+ + ") ENGINE=OLAP\n"
+ + "UNIQUE KEY(`key`)\n"
+ + "DISTRIBUTED BY HASH(`key`) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + " \"replication_num\" = \"1\",\n"
+ + " \"enable_unique_key_merge_on_write\" = \"true\",\n"
+ + " \"light_schema_change\" = \"true\",\n"
+ + " \"store_row_column\" = \"true\",\n"
+ + " \"changelog.enable\" = \"true\"\n"
+ + ");");
+ }
+
+ @Test
+ void testHiddenColumnsDoNotUseShortCircuit() {
+ // The row store returned 0 for __DORIS_VERSION_COL__ and
__DORIS_COMMIT_TSO_COL__ instead
+ // of the rowset version and commit TSO.
+ rewrite("select __DORIS_VERSION_COL__ from tbl_changelog_point_query
where `key` = 1");
+
Assertions.assertFalse(connectContext.getStatementContext().isShortCircuitQuery());
+
+ rewrite("select __DORIS_COMMIT_TSO_COL__, v1 from
tbl_changelog_point_query where `key` = 1");
+
Assertions.assertFalse(connectContext.getStatementContext().isShortCircuitQuery());
+
+ rewrite("select __DORIS_COMMIT_TSO_COL__ + 1 from
tbl_changelog_point_query where `key` = 1");
+
Assertions.assertFalse(connectContext.getStatementContext().isShortCircuitQuery());
+
+ rewrite("select __DORIS_DELETE_SIGN__, v1 from
tbl_changelog_point_query where `key` = 1");
+
Assertions.assertFalse(connectContext.getStatementContext().isShortCircuitQuery());
+
+ // show_hidden_columns makes `select *` return hidden columns as well.
+ boolean originalShowHidden =
connectContext.getSessionVariable().showHiddenColumns();
+ try {
+ connectContext.getSessionVariable().setShowHiddenColumns(true);
+ rewrite("select * from tbl_changelog_point_query where `key` = 1");
+
Assertions.assertFalse(connectContext.getStatementContext().isShortCircuitQuery());
+ } finally {
+
connectContext.getSessionVariable().setShowHiddenColumns(originalShowHidden);
+ }
+ }
+
+ @Test
+ void testQueryWithoutHiddenColumnsUsesShortCircuit() {
+ // `select *` skips hidden columns by default.
+ rewrite("select * from tbl_changelog_point_query where `key` = 1");
+
Assertions.assertTrue(connectContext.getStatementContext().isShortCircuitQuery());
+
+ rewrite("select v1 from tbl_changelog_point_query where `key` = 1");
+
Assertions.assertTrue(connectContext.getStatementContext().isShortCircuitQuery());
}
@Test
diff --git
a/regression-test/data/point_query_p0/test_short_circuit_rowset_hidden_columns.out
b/regression-test/data/point_query_p0/test_short_circuit_rowset_hidden_columns.out
new file mode 100644
index 00000000000..d7cbd41dd1b
--- /dev/null
+++
b/regression-test/data/point_query_p0/test_short_circuit_rowset_hidden_columns.out
@@ -0,0 +1,10 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !version --
+2 7
+
+-- !commit_tso --
+true 7
+
+-- !select_star --
+1 7
+
diff --git
a/regression-test/suites/point_query_p0/test_short_circuit_rowset_hidden_columns.groovy
b/regression-test/suites/point_query_p0/test_short_circuit_rowset_hidden_columns.groovy
new file mode 100644
index 00000000000..e9755540b03
--- /dev/null
+++
b/regression-test/suites/point_query_p0/test_short_circuit_rowset_hidden_columns.groovy
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_short_circuit_rowset_hidden_columns", "p0") {
+ // Queries reading hidden columns do not use the short circuit: it used to
return 0 for
+ // __DORIS_VERSION_COL__ and __DORIS_COMMIT_TSO_COL__ instead of version 2
and the commit TSO.
+ sql "DROP TABLE IF EXISTS short_circuit_rowset_hidden_columns"
+ sql """
+ CREATE TABLE short_circuit_rowset_hidden_columns (
+ `id` INT NOT NULL,
+ `v` INT NULL
+ ) ENGINE=OLAP
+ UNIQUE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true",
+ "light_schema_change" = "true",
+ "store_row_column" = "true",
+ "changelog.enable" = "true"
+ )
+ """
+ sql "INSERT INTO short_circuit_rowset_hidden_columns VALUES (1, 7), (2, 8)"
+
+ explain {
+ sql "SELECT __DORIS_VERSION_COL__, v FROM
short_circuit_rowset_hidden_columns WHERE id = 1"
+ notContains "SHORT-CIRCUIT"
+ }
+ qt_version "SELECT __DORIS_VERSION_COL__, v FROM
short_circuit_rowset_hidden_columns WHERE id = 1"
+
+ explain {
+ sql "SELECT __DORIS_COMMIT_TSO_COL__ > 0, v FROM
short_circuit_rowset_hidden_columns WHERE id = 1"
+ notContains "SHORT-CIRCUIT"
+ }
+ qt_commit_tso "SELECT __DORIS_COMMIT_TSO_COL__ > 0, v FROM
short_circuit_rowset_hidden_columns WHERE id = 1"
+
+ // show_hidden_columns makes `select *` return hidden columns as well.
+ sql "SET show_hidden_columns = true"
+ explain {
+ sql "SELECT * FROM short_circuit_rowset_hidden_columns WHERE id = 1"
+ notContains "SHORT-CIRCUIT"
+ }
+ sql "SET show_hidden_columns = false"
+
+ // Queries that do not read hidden columns keep the short circuit.
+ explain {
+ sql "SELECT * FROM short_circuit_rowset_hidden_columns WHERE id = 1"
+ contains "SHORT-CIRCUIT"
+ }
+ qt_select_star "SELECT * FROM short_circuit_rowset_hidden_columns WHERE id
= 1"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
