This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 91d72f6efad # [fix](topn lazy materialization) Disable topn lazy
materialization on non-light-schema-change tables (#65415)
91d72f6efad is described below
commit 91d72f6efadaa703aba53b98a8eca7524f9b4c91
Author: Lijia Liu <[email protected]>
AuthorDate: Fri Jul 24 14:51:24 2026 +0800
# [fix](topn lazy materialization) Disable topn lazy materialization on
non-light-schema-change tables (#65415)
A top-N query that emits a non-order-by column fails on a
**non-light-schema-change** OLAP table (a table created/upgraded with
`light_schema_change = false`, where every column's `uniqueId` is `-1`):
```sql
select id, name from tbl order by createdate desc limit 10;
```
```
ERROR 1105 (HY000): errCode = 2, detailMessage =
[INTERNAL_ERROR]field name is invalid. field=__DORIS_GLOBAL_ROWID_COL__tbl,
field_name_to_index=[...], col_unique_id=2147483647
```
Fix: disable topn lazy materialization for non-light-schema-change OLAP
tables in MaterializeProbeVisitor and fall back to normal topn (the
working two-phase read path). A new helper
supportOlapTopnLazyMaterialize() consolidates the existing AGG_KEYS
exclusion with the new light_schema_change requirement, applied at
visitPhysicalOlapScan, visitPhysicalCatalogRelation and
visitPhysicalFilter.
Add regression test topn_lazy_light_schema_change verifying:
- light_schema_change=false: no lazy materialization in the plan,
correct results.
- light_schema_change=true: lazy materialization still applies, correct
results.
## Behavior after the fix
| Table | Plan | Result |
|-------|------|--------|
| `light_schema_change = false` | plain `PhysicalOlapScan` (no lazy) →
safe two-phase read | correct values |
| `light_schema_change = true` | `PhysicalLazyMaterialize` /
`PhysicalLazyMaterializeOlapScan` (unchanged) | correct values |
---
.../post/materialize/MaterializeProbeVisitor.java | 36 ++++++-
.../materialize/MaterializeProbeVisitorTest.java | 2 +
.../topn_lazy/topn_lazy_light_schema_change.out | 11 ++
.../topn_lazy/topn_lazy_light_schema_change.groovy | 119 +++++++++++++++++++++
4 files changed, 164 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java
index 0a497896600..9bcdedb01e6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java
@@ -87,9 +87,9 @@ public class MaterializeProbeVisitor extends
DefaultPlanVisitor<Optional<Materia
public Optional<MaterializeSource> visitPhysicalFilter(PhysicalFilter<?
extends Plan> filter,
ProbeContext
context) {
if (SessionVariable.getTopNLazyMaterializationUsingIndex() &&
filter.child() instanceof PhysicalOlapScan) {
- // agg table do not support lazy materialize
+ // agg table / non-light-schema-change table do not support lazy
materialize
OlapTable table = ((PhysicalOlapScan) filter.child()).getTable();
- if (KeysType.AGG_KEYS.equals(table.getKeysType())) {
+ if (!supportOlapTopnLazyMaterialize(table)) {
return Optional.empty();
}
if (filter.getInputSlots().contains(context.slot)) {
@@ -133,6 +133,34 @@ public class MaterializeProbeVisitor extends
DefaultPlanVisitor<Optional<Materia
return (hmsExternalTable.getDlaType() == DLAType.HIVE &&
hmsExternalTable.supportedHiveTopNLazyTable())
|| hmsExternalTable.getDlaType() == DLAType.ICEBERG;
}
+ if (relation.getTable() instanceof OlapTable) {
+ return supportOlapTopnLazyMaterialize((OlapTable)
relation.getTable());
+ }
+ return true;
+ }
+
+ /**
+ * Whether an OLAP table can perform topn lazy materialization.
+ *
+ * <p>Two hard requirements:
+ * <ul>
+ * <li>Not an AGG_KEYS table: aggregate tables cannot locate a single
source row for a value.</li>
+ * <li>light_schema_change is enabled: lazy materialization appends a
synthetic global row-id
+ * column to the scan and relies on the BE rebuilding the tablet
schema from FE's
+ * columns_desc (keyed by column uniqueId). For
non-light-schema-change tables every
+ * column's uniqueId is -1, so the BE keeps its own on-disk schema
and never installs the
+ * synthetic row-id column. That path either fails with "field name
is invalid" during the
+ * scan or silently returns NULL for the lazily-fetched columns.
Disable the optimization
+ * for such tables and fall back to normal topn.</li>
+ * </ul>
+ */
+ private boolean supportOlapTopnLazyMaterialize(OlapTable table) {
+ if (KeysType.AGG_KEYS.equals(table.getKeysType())) {
+ return false;
+ }
+ if (!table.getEnableLightSchemaChange()) {
+ return false;
+ }
return true;
}
@@ -155,9 +183,9 @@ public class MaterializeProbeVisitor extends
DefaultPlanVisitor<Optional<Materia
if (scan.getSelectedIndexId() != scan.getTable().getBaseIndexId()) {
return Optional.empty();
}
- // agg table do not support lazy materialize
+ // agg table / non-light-schema-change table do not support lazy
materialize
OlapTable table = scan.getTable();
- if (KeysType.AGG_KEYS.equals(table.getKeysType())) {
+ if (!supportOlapTopnLazyMaterialize(table)) {
return Optional.empty();
}
if (context.requiredMaterializedSlots.contains(context.slot)) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitorTest.java
index 100c70cfb4e..1ab7ddafdbc 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitorTest.java
@@ -160,6 +160,8 @@ public class MaterializeProbeVisitorTest {
OlapTable table = Mockito.mock(OlapTable.class);
Mockito.when(table.getBaseIndexId()).thenReturn(1L);
Mockito.when(table.getKeysType()).thenReturn(KeysType.DUP_KEYS);
+ // lazy materialization requires light_schema_change to be enabled
+ Mockito.when(table.getEnableLightSchemaChange()).thenReturn(true);
PhysicalOlapScan scan = Mockito.mock(PhysicalOlapScan.class);
Mockito.when(scan.getSelectedIndexId()).thenReturn(1L);
Mockito.when(scan.getTable()).thenReturn(table);
diff --git
a/regression-test/data/query_p0/topn_lazy/topn_lazy_light_schema_change.out
b/regression-test/data/query_p0/topn_lazy/topn_lazy_light_schema_change.out
new file mode 100644
index 00000000000..878dce10f7b
--- /dev/null
+++ b/regression-test/data/query_p0/topn_lazy/topn_lazy_light_schema_change.out
@@ -0,0 +1,11 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !result_lsc_false --
+5 eee
+4 ddd
+3 ccc
+
+-- !result_lsc_true --
+5 eee
+4 ddd
+3 ccc
+
diff --git
a/regression-test/suites/query_p0/topn_lazy/topn_lazy_light_schema_change.groovy
b/regression-test/suites/query_p0/topn_lazy/topn_lazy_light_schema_change.groovy
new file mode 100644
index 00000000000..8d76c476a25
--- /dev/null
+++
b/regression-test/suites/query_p0/topn_lazy/topn_lazy_light_schema_change.groovy
@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Regression test for topn lazy materialization on non-light-schema-change
tables.
+//
+// TopN lazy materialization appends a synthetic global row-id column
+// (__DORIS_GLOBAL_ROWID_COL__<table>) to the OLAP scan. The BE only rebuilds
the
+// tablet schema from FE's columns_desc when columns_desc[0].col_unique_id >=
0,
+// which is only true for light_schema_change tables. On a
non-light-schema-change
+// table every column's uniqueId is -1, so the synthetic row-id column never
enters
+// the BE tablet schema and the scan fails with
+// "field name is invalid. field=__DORIS_GLOBAL_ROWID_COL__<table>"
+// (or, if the scan schema is forced, the second-phase fetch silently returns
NULL
+// for the lazily-materialized columns).
+//
+// The fix disables topn lazy materialization for non-light-schema-change OLAP
tables
+// in MaterializeProbeVisitor, falling back to normal topn. This test verifies
both
+// the plan shape (no lazy materialization) and correct results on such a
table, and
+// verifies that a light_schema_change=true table still uses lazy
materialization.
+suite("topn_lazy_light_schema_change") {
+ // ---- light_schema_change = false : lazy materialization must be
disabled ----
+ sql """ drop table if exists topn_lazy_lsc_false """
+ sql """
+ CREATE TABLE topn_lazy_lsc_false (
+ `id` INT NOT NULL,
+ `name` VARCHAR(64),
+ `createdate` DATETIME
+ )
+ UNIQUE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 3
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "light_schema_change" = "false",
+ "enable_unique_key_merge_on_write" = "false"
+ );
+ """
+ sql """
+ insert into topn_lazy_lsc_false values
+ (1, 'aaa', '2024-01-01 10:00:00'),
+ (2, 'bbb', '2024-01-02 10:00:00'),
+ (3, 'ccc', '2024-01-03 10:00:00'),
+ (4, 'ddd', '2024-01-04 10:00:00'),
+ (5, 'eee', '2024-01-05 10:00:00');
+ """
+
+ // Plan shape: a non-light-schema-change table must fall back to a plain
+ // PhysicalOlapScan with no PhysicalLazyMaterialize /
PhysicalLazyMaterializeOlapScan.
+ //
+ // Only meaningful in storage-compute integrated (non-cloud) mode. In
cloud mode
+ // CloudPropertyAnalyzer force-rewrites light_schema_change to "true"
+ // (RewriteProperty.replace), so this table is effectively
light_schema_change=true
+ // and lazy materialization legitimately applies -- the assertion would
not hold.
+ if (!isCloudMode()) {
+ explain {
+ sql "shape plan select name from topn_lazy_lsc_false order by
createdate desc limit 3"
+ notContains("PhysicalLazyMaterialize")
+ }
+ }
+
+ // Correctness: the query used to error with "field name is invalid" (or
return NULL
+ // for name). It must now run and return the real values ordered by
createdate desc.
+ qt_result_lsc_false """
+ select id, name from topn_lazy_lsc_false order by createdate desc
limit 3
+ """
+
+ // ---- light_schema_change = true : lazy materialization must still apply
----
+ sql """ drop table if exists topn_lazy_lsc_true """
+ sql """
+ CREATE TABLE topn_lazy_lsc_true (
+ `id` INT NOT NULL,
+ `name` VARCHAR(64),
+ `createdate` DATETIME
+ )
+ UNIQUE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 3
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "light_schema_change" = "true",
+ "enable_unique_key_merge_on_write" = "false"
+ );
+ """
+ sql """
+ insert into topn_lazy_lsc_true values
+ (1, 'aaa', '2024-01-01 10:00:00'),
+ (2, 'bbb', '2024-01-02 10:00:00'),
+ (3, 'ccc', '2024-01-03 10:00:00'),
+ (4, 'ddd', '2024-01-04 10:00:00'),
+ (5, 'eee', '2024-01-05 10:00:00');
+ """
+
+ // Plan shape: a light_schema_change table keeps lazy materialization
+ // (PhysicalLazyMaterialize present). Checked only in non-cloud mode, for
symmetry
+ // with the lsc_false case above.
+ if (!isCloudMode()) {
+ explain {
+ sql "shape plan select name from topn_lazy_lsc_true order by
createdate desc limit 3"
+ contains("PhysicalLazyMaterialize")
+ }
+ }
+
+ // Correctness: lazy materialization returns the same real values.
+ qt_result_lsc_true """
+ select id, name from topn_lazy_lsc_true order by createdate desc limit
3
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
