This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new a514f2b1bd0 [fix](topn lazy materialization) Disable topn lazy
materialization on non-light-schema-change tables (#65415) (#67928)
a514f2b1bd0 is described below
commit a514f2b1bd095d09dd45ed331cda0c509d233906
Author: Lijia Liu <[email protected]>
AuthorDate: Tue Sep 15 12:00:13 2026 +0800
[fix](topn lazy materialization) Disable topn lazy materialization on
non-light-schema-change tables (#65415) (#67928)
bp #65415
---
.../post/materialize/MaterializeProbeVisitor.java | 36 ++++++-
.../materialize/MaterializeProbeVisitorTest.java | 2 +
.../topn_lazy/topn_lazy_light_schema_change.out | 11 ++
.../topn_lazy/topn_lazy_light_schema_change.groovy | 119 +++++++++++++++++++++
4 files changed, 164 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java
index 702bc486147..177c697df4f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java
@@ -80,9 +80,9 @@ public class MaterializeProbeVisitor extends
DefaultPlanVisitor<Optional<Materia
public Optional<MaterializeSource> visitPhysicalFilter(PhysicalFilter<?
extends Plan> filter,
ProbeContext
context) {
if (SessionVariable.getTopNLazyMaterializationUsingIndex() &&
filter.child() instanceof PhysicalOlapScan) {
- // agg table do not support lazy materialize
+ // agg table / non-light-schema-change table do not support lazy
materialize
OlapTable table = ((PhysicalOlapScan) filter.child()).getTable();
- if (KeysType.AGG_KEYS.equals(table.getKeysType())) {
+ if (!supportOlapTopnLazyMaterialize(table)) {
return Optional.empty();
}
if (filter.getInputSlots().contains(context.slot)) {
@@ -126,6 +126,34 @@ public class MaterializeProbeVisitor extends
DefaultPlanVisitor<Optional<Materia
return (hmsExternalTable.getDlaType() == DLAType.HIVE &&
hmsExternalTable.supportedHiveTopNLazyTable())
|| hmsExternalTable.getDlaType() == DLAType.ICEBERG;
}
+ if (relation.getTable() instanceof OlapTable) {
+ return supportOlapTopnLazyMaterialize((OlapTable)
relation.getTable());
+ }
+ return true;
+ }
+
+ /**
+ * Whether an OLAP table can perform topn lazy materialization.
+ *
+ * <p>Two hard requirements:
+ * <ul>
+ * <li>Not an AGG_KEYS table: aggregate tables cannot locate a single
source row for a value.</li>
+ * <li>light_schema_change is enabled: lazy materialization appends a
synthetic global row-id
+ * column to the scan and relies on the BE rebuilding the tablet
schema from FE's
+ * columns_desc (keyed by column uniqueId). For
non-light-schema-change tables every
+ * column's uniqueId is -1, so the BE keeps its own on-disk schema
and never installs the
+ * synthetic row-id column. That path either fails with "field name
is invalid" during the
+ * scan or silently returns NULL for the lazily-fetched columns.
Disable the optimization
+ * for such tables and fall back to normal topn.</li>
+ * </ul>
+ */
+ private boolean supportOlapTopnLazyMaterialize(OlapTable table) {
+ if (KeysType.AGG_KEYS.equals(table.getKeysType())) {
+ return false;
+ }
+ if (!table.getEnableLightSchemaChange()) {
+ return false;
+ }
return true;
}
@@ -158,9 +186,9 @@ public class MaterializeProbeVisitor extends
DefaultPlanVisitor<Optional<Materia
if (scan.getSelectedIndexId() != scan.getTable().getBaseIndexId()) {
return Optional.empty();
}
- // agg table do not support lazy materialize
+ // agg table / non-light-schema-change table do not support lazy
materialize
OlapTable table = scan.getTable();
- if (KeysType.AGG_KEYS.equals(table.getKeysType())) {
+ if (!supportOlapTopnLazyMaterialize(table)) {
return Optional.empty();
}
if (scan.getOperativeSlots().contains(context.slot)) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitorTest.java
index 94d24536693..f37806b9c6e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitorTest.java
@@ -167,6 +167,8 @@ class MaterializeProbeVisitorTest {
OlapTable table = Mockito.mock(OlapTable.class);
Mockito.when(table.getBaseIndexId()).thenReturn(1L);
Mockito.when(table.getKeysType()).thenReturn(KeysType.DUP_KEYS);
+ // lazy materialization requires light_schema_change to be enabled
+ Mockito.when(table.getEnableLightSchemaChange()).thenReturn(true);
PhysicalOlapScan scan = Mockito.mock(PhysicalOlapScan.class);
Mockito.when(scan.getSelectedIndexId()).thenReturn(1L);
Mockito.when(scan.getTable()).thenReturn(table);
diff --git
a/regression-test/data/query_p0/topn_lazy/topn_lazy_light_schema_change.out
b/regression-test/data/query_p0/topn_lazy/topn_lazy_light_schema_change.out
new file mode 100644
index 00000000000..878dce10f7b
--- /dev/null
+++ b/regression-test/data/query_p0/topn_lazy/topn_lazy_light_schema_change.out
@@ -0,0 +1,11 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !result_lsc_false --
+5 eee
+4 ddd
+3 ccc
+
+-- !result_lsc_true --
+5 eee
+4 ddd
+3 ccc
+
diff --git
a/regression-test/suites/query_p0/topn_lazy/topn_lazy_light_schema_change.groovy
b/regression-test/suites/query_p0/topn_lazy/topn_lazy_light_schema_change.groovy
new file mode 100644
index 00000000000..8d76c476a25
--- /dev/null
+++
b/regression-test/suites/query_p0/topn_lazy/topn_lazy_light_schema_change.groovy
@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Regression test for topn lazy materialization on non-light-schema-change
tables.
+//
+// TopN lazy materialization appends a synthetic global row-id column
+// (__DORIS_GLOBAL_ROWID_COL__<table>) to the OLAP scan. The BE only rebuilds
the
+// tablet schema from FE's columns_desc when columns_desc[0].col_unique_id >=
0,
+// which is only true for light_schema_change tables. On a
non-light-schema-change
+// table every column's uniqueId is -1, so the synthetic row-id column never
enters
+// the BE tablet schema and the scan fails with
+// "field name is invalid. field=__DORIS_GLOBAL_ROWID_COL__<table>"
+// (or, if the scan schema is forced, the second-phase fetch silently returns
NULL
+// for the lazily-materialized columns).
+//
+// The fix disables topn lazy materialization for non-light-schema-change OLAP
tables
+// in MaterializeProbeVisitor, falling back to normal topn. This test verifies
both
+// the plan shape (no lazy materialization) and correct results on such a
table, and
+// verifies that a light_schema_change=true table still uses lazy
materialization.
+suite("topn_lazy_light_schema_change") {
+ // ---- light_schema_change = false : lazy materialization must be
disabled ----
+ sql """ drop table if exists topn_lazy_lsc_false """
+ sql """
+ CREATE TABLE topn_lazy_lsc_false (
+ `id` INT NOT NULL,
+ `name` VARCHAR(64),
+ `createdate` DATETIME
+ )
+ UNIQUE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 3
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "light_schema_change" = "false",
+ "enable_unique_key_merge_on_write" = "false"
+ );
+ """
+ sql """
+ insert into topn_lazy_lsc_false values
+ (1, 'aaa', '2024-01-01 10:00:00'),
+ (2, 'bbb', '2024-01-02 10:00:00'),
+ (3, 'ccc', '2024-01-03 10:00:00'),
+ (4, 'ddd', '2024-01-04 10:00:00'),
+ (5, 'eee', '2024-01-05 10:00:00');
+ """
+
+ // Plan shape: a non-light-schema-change table must fall back to a plain
+ // PhysicalOlapScan with no PhysicalLazyMaterialize /
PhysicalLazyMaterializeOlapScan.
+ //
+ // Only meaningful in storage-compute integrated (non-cloud) mode. In
cloud mode
+ // CloudPropertyAnalyzer force-rewrites light_schema_change to "true"
+ // (RewriteProperty.replace), so this table is effectively
light_schema_change=true
+ // and lazy materialization legitimately applies -- the assertion would
not hold.
+ if (!isCloudMode()) {
+ explain {
+ sql "shape plan select name from topn_lazy_lsc_false order by
createdate desc limit 3"
+ notContains("PhysicalLazyMaterialize")
+ }
+ }
+
+ // Correctness: the query used to error with "field name is invalid" (or
return NULL
+ // for name). It must now run and return the real values ordered by
createdate desc.
+ qt_result_lsc_false """
+ select id, name from topn_lazy_lsc_false order by createdate desc
limit 3
+ """
+
+ // ---- light_schema_change = true : lazy materialization must still apply
----
+ sql """ drop table if exists topn_lazy_lsc_true """
+ sql """
+ CREATE TABLE topn_lazy_lsc_true (
+ `id` INT NOT NULL,
+ `name` VARCHAR(64),
+ `createdate` DATETIME
+ )
+ UNIQUE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 3
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "light_schema_change" = "true",
+ "enable_unique_key_merge_on_write" = "false"
+ );
+ """
+ sql """
+ insert into topn_lazy_lsc_true values
+ (1, 'aaa', '2024-01-01 10:00:00'),
+ (2, 'bbb', '2024-01-02 10:00:00'),
+ (3, 'ccc', '2024-01-03 10:00:00'),
+ (4, 'ddd', '2024-01-04 10:00:00'),
+ (5, 'eee', '2024-01-05 10:00:00');
+ """
+
+ // Plan shape: a light_schema_change table keeps lazy materialization
+ // (PhysicalLazyMaterialize present). Checked only in non-cloud mode, for
symmetry
+ // with the lsc_false case above.
+ if (!isCloudMode()) {
+ explain {
+ sql "shape plan select name from topn_lazy_lsc_true order by
createdate desc limit 3"
+ contains("PhysicalLazyMaterialize")
+ }
+ }
+
+ // Correctness: lazy materialization returns the same real values.
+ qt_result_lsc_true """
+ select id, name from topn_lazy_lsc_true order by createdate desc limit
3
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
