This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 08203ba4140 [fix](mtmv) Fix query err when calc mv functional
dependency which has variant and log more detailed info for troubleshoot a
problem (#59933)
08203ba4140 is described below
commit 08203ba414012d4ee97ad8a5859626dae839a877
Author: seawinde <[email protected]>
AuthorDate: Tue Mar 24 14:23:19 2026 +0800
[fix](mtmv) Fix query err when calc mv functional dependency which has
variant and log more detailed info for troubleshoot a problem (#59933)
Fix query err when calc mv fd by catch it and log more detailed info for
troubleshoot a problem
1. originOutputs.size() should equlas targetOutputs.size(), if not would
throw exception, this would cause query err,
should log err log and not cause query err
2. current log could not find the root cause, so add detail log
3. fix the problem by mapping the slot full path name between scan mv
output and mv sql plan output
for example as following
```sql
CREATE TABLE fact_var (
k INT,
v VARIANT
) ENGINE=OLAP
DUPLICATE KEY(k)
DISTRIBUTED BY HASH(k) BUCKETS 1
PROPERTIES ("replication_num" = "1");
INSERT INTO fact_var VALUES
(1, '{"a":1,"b":{"c":10,"d":20}}'),
(2, '{"a":2,"b":{"c":30}}');
CREATE MATERIALIZED VIEW mv_var
BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
DISTRIBUTED BY RANDOM BUCKETS 2
PROPERTIES ('replication_num' = '1')
AS
SELECT k, v
FROM fact_var;
```
if run query as fllowing, mv_var sacn would return k, v, v['a'] but
mv_var def plan sql, would return k, v,
the size is different but also can work after the fix
```sql
SELECT v['a'] AS c_val FROM mv_var;
```
Related PR: #40106
---
.../trees/plans/logical/LogicalOlapScan.java | 70 +++++--
.../trees/plans/logical/LogicalOlapScanTest.java | 222 +++++++++++++++++++++
2 files changed, 278 insertions(+), 14 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
index 13d055d01be..2186445d189 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
@@ -63,8 +63,10 @@ import org.json.JSONObject;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
@@ -903,10 +905,9 @@ public class LogicalOlapScan extends
LogicalCatalogRelation implements OlapScan,
scoreRangeInfo, annOrderKeys, annLimit, tableAlias));
}
- private Map<Slot, Slot> constructReplaceMap(MTMV mtmv) {
+ @VisibleForTesting
+ Map<Slot, Slot> constructReplaceMap(MTMV mtmv) {
Map<Slot, Slot> replaceMap = new HashMap<>();
- // Need remove invisible column, and then mapping them
- List<Slot> originOutputs = new ArrayList<>();
MTMVCache cache;
try {
cache = mtmv.getOrGenerateCache(ConnectContext.get());
@@ -914,21 +915,62 @@ public class LogicalOlapScan extends
LogicalCatalogRelation implements OlapScan,
LOG.warn(String.format("LogicalOlapScan constructReplaceMap fail,
mv name is %s", mtmv.getName()), e);
return replaceMap;
}
- for (Slot originSlot : cache.getOriginalFinalPlan().getOutput()) {
- if (!(originSlot instanceof SlotReference) || (((SlotReference)
originSlot).isVisible())) {
- originOutputs.add(originSlot);
+ // Get MV plan's visible output slots (ordered, matching MV definition
SELECT list).
+ // This includes all visible slots: regular columns AND variant
subPath columns
+ // like payload['issue'] that are real physical columns of the MV.
+ List<Slot> mvPlanVisibleOutputs = new ArrayList<>();
+ for (Slot slot : cache.getOriginalFinalPlan().getOutput()) {
+ if (slot instanceof SlotReference && ((SlotReference)
slot).isVisible()) {
+ mvPlanVisibleOutputs.add(slot);
}
}
- List<Slot> targetOutputs = new ArrayList<>();
- for (Slot targeSlot : getOutput()) {
- if (!(targeSlot instanceof SlotReference) || (((SlotReference)
targeSlot).isVisible())) {
- targetOutputs.add(targeSlot);
+ // Get MV table's visible physical columns (ordered).
+ // getBaseSchema() returns visible-only columns whose names are
derived from the
+ // CREATE MV AS SELECT aliases. These names are guaranteed unique per
table.
+ // Using physical column names as keys (instead of plan slot's
originalColumn.getName())
+ // correctly handles:
+ // - Aliased columns (e.g. SELECT sum_total AS agg3): key is "agg3",
not "sum_total"
+ // - Self-join MVs: physical column names are unique even if source
columns collide
+ // - Variant columns (e.g. SELECT payload['issue']): they are physical
columns in getBaseSchema()
+ List<Column> mvPhysicalColumns = mtmv.getBaseSchema();
+ if (mvPlanVisibleOutputs.size() != mvPhysicalColumns.size()) {
+ LOG.error("LogicalOlapScan constructReplaceMap: MV plan visible
output size {} "
+ + "doesn't match physical column size {} for mv {}",
+ mvPlanVisibleOutputs.size(), mvPhysicalColumns.size(),
mtmv.getName());
+ // not throw exception here to avoid query failed, compute mv fd
should not influence query process
+ return Collections.emptyMap();
+ }
+ // Build mvOutputsMap: the i-th visible plan output corresponds to the
i-th physical column.
+ Map<List<String>, Slot> mvOutputsMap = new HashMap<>();
+ for (int i = 0; i < mvPlanVisibleOutputs.size(); i++) {
+ String physicalColName =
mvPhysicalColumns.get(i).getName().toLowerCase(Locale.ROOT);
+ List<String> key = Lists.newArrayList(physicalColName);
+ mvOutputsMap.put(key, mvPlanVisibleOutputs.get(i));
+ }
+ // Match scan output slots against mvOutputsMap.
+ // Scan slot's originalColumn.getName() refers to the MV's physical
column, so keys match.
+ // Extra subPath slots added by VariantSubPathPruning during query
optimization won't
+ // match any mvOutputsMap entry (their keys include subPath elements)
and are safely skipped.
+ for (Slot scanSlot : getOutput()) {
+ if (scanSlot instanceof SlotReference && ((SlotReference)
scanSlot).isVisible()) {
+ SlotReference scanRef = (SlotReference) scanSlot;
+ String scanName =
scanRef.getOriginalColumn().map(Column::getName).orElse(scanRef.getName());
+ List<String> key =
Lists.newArrayList(scanName.toLowerCase(Locale.ROOT));
+ key.addAll(scanRef.getSubPath());
+ Slot mvMappingSlot = mvOutputsMap.get(key);
+ if (mvMappingSlot != null) {
+ replaceMap.put(mvMappingSlot, scanSlot);
+ }
}
}
- Preconditions.checkArgument(originOutputs.size() ==
targetOutputs.size(),
- "constructReplaceMap, the size of originOutputs and
targetOutputs should be same");
- for (int i = 0; i < targetOutputs.size(); i++) {
- replaceMap.put(originOutputs.get(i), targetOutputs.get(i));
+ // Every MV plan slot must be mapped
+ if (mvOutputsMap.size() != replaceMap.size()) {
+ LOG.error(String.format("LogicalOlapScan constructReplaceMap size
not match,"
+ + "mv name is %s, mvOutputsMap is %s, mv output is %s,
scan output is %s",
+ mtmv.getName(), mvOutputsMap,
+ cache.getOriginalFinalPlan().getOutput(), getOutput()));
+ // not throw exception here to avoid query failed, compute mv fd
should not influence query process
+ return Collections.emptyMap();
}
return replaceMap;
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScanTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScanTest.java
new file mode 100644
index 00000000000..32ecb65ee98
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScanTest.java
@@ -0,0 +1,222 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.logical;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.MTMV;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.mtmv.MTMVCache;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.RelationId;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Tests for LogicalOlapScan
+ */
+public class LogicalOlapScanTest {
+
+ private ConnectContext connectContext;
+ private MockedStatic<ConnectContext> mockedConnectContext;
+
+ @BeforeEach
+ public void setUp() {
+ connectContext = new ConnectContext();
+ connectContext.setSessionVariable(new SessionVariable());
+ mockedConnectContext = Mockito.mockStatic(ConnectContext.class);
+
mockedConnectContext.when(ConnectContext::get).thenReturn(connectContext);
+ }
+
+ @AfterEach
+ public void tearDown() {
+ if (mockedConnectContext != null) {
+ mockedConnectContext.close();
+ }
+ }
+
+ private SlotReference createMockSlot(String name, boolean isVisible) {
+ SlotReference slot = Mockito.mock(SlotReference.class);
+ Mockito.when(slot.isVisible()).thenReturn(isVisible);
+ Mockito.when(slot.getName()).thenReturn(name);
+ Mockito.when(slot.getSubPath()).thenReturn(Collections.emptyList());
+ Mockito.when(slot.getOriginalColumn()).thenReturn(Optional.empty());
+ return slot;
+ }
+
+ private SlotReference createMockSlot(String name, String
originalColumnName,
+ List<String> subPath, boolean isVisible) {
+ SlotReference slot = Mockito.mock(SlotReference.class);
+ Column column = createColumn(originalColumnName);
+ Mockito.when(slot.isVisible()).thenReturn(isVisible);
+ Mockito.when(slot.getName()).thenReturn(name);
+ Mockito.when(slot.getSubPath()).thenReturn(subPath);
+ Mockito.when(slot.getOriginalColumn()).thenReturn(Optional.of(column));
+ return slot;
+ }
+
+ private Column createColumn(String name) {
+ return new Column(name, PrimitiveType.INT);
+ }
+
+ private LogicalOlapScan createMockScan(List<Slot> outputSlots) {
+ OlapTable olapTable = Mockito.mock(OlapTable.class);
+ Mockito.when(olapTable.getId()).thenReturn(1L);
+ Mockito.when(olapTable.getName()).thenReturn("test_table");
+
Mockito.when(olapTable.getFullQualifiers()).thenReturn(ImmutableList.of("db",
"test_table"));
+
+ LogicalOlapScan scan = Mockito.spy(new LogicalOlapScan(
+ new RelationId(1),
+ olapTable,
+ ImmutableList.of("db"),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Optional.empty(),
+ Collections.emptyList()
+ ));
+ Mockito.doReturn(outputSlots).when(scan).getOutput();
+ return scan;
+ }
+
+ /**
+ * Test constructReplaceMap returns empty map when MV plan output size
+ * doesn't match physical column size.
+ */
+ @Test
+ public void testConstructReplaceMapSizeMismatch() throws Exception {
+ MTMV mtmv = Mockito.mock(MTMV.class);
+ MTMVCache cache = Mockito.mock(MTMVCache.class);
+ Plan originalPlan = Mockito.mock(Plan.class);
+
+ // MV plan has 3 visible output slots
+ List<Slot> originOutputs = ImmutableList.of(
+ createMockSlot("col1", true),
+ createMockSlot("col2", true),
+ createMockSlot("col3", true));
+
+ Mockito.when(mtmv.getOrGenerateCache(Mockito.any())).thenReturn(cache);
+ Mockito.when(mtmv.getName()).thenReturn("test_mv");
+ Mockito.when(cache.getOriginalFinalPlan()).thenReturn(originalPlan);
+ Mockito.when(originalPlan.getOutput()).thenReturn(originOutputs);
+
+ // But MV physical table only has 2 columns (size mismatch with plan
output)
+ Mockito.when(mtmv.getBaseSchema()).thenReturn(ImmutableList.of(
+ createColumn("col1"), createColumn("col2")));
+
+ LogicalOlapScan scan = createMockScan(ImmutableList.of(
+ createMockSlot("col1", "col1", Collections.emptyList(), true),
+ createMockSlot("col2", "col2", Collections.emptyList(),
true)));
+
+ Map<Slot, Slot> replaceMap = scan.constructReplaceMap(mtmv);
+
+ Assertions.assertTrue(replaceMap.isEmpty(),
+ "replaceMap should be empty when plan output size doesn't
match physical column size");
+ }
+
+ /**
+ * Test constructReplaceMap ignores extra subPath slots in scan output
+ * (added by VariantSubPathPruning during query optimization).
+ */
+ @Test
+ public void testConstructReplaceMapIgnoresExtraScanSlots() throws
Exception {
+ MTMV mtmv = Mockito.mock(MTMV.class);
+ MTMVCache cache = Mockito.mock(MTMVCache.class);
+ Plan originalPlan = Mockito.mock(Plan.class);
+
+ SlotReference mvSlot = createMockSlot("col1", true);
+ Mockito.when(mtmv.getOrGenerateCache(Mockito.any())).thenReturn(cache);
+ Mockito.when(mtmv.getName()).thenReturn("test_mv");
+ Mockito.when(cache.getOriginalFinalPlan()).thenReturn(originalPlan);
+
Mockito.when(originalPlan.getOutput()).thenReturn(ImmutableList.of(mvSlot));
+
Mockito.when(mtmv.getBaseSchema()).thenReturn(ImmutableList.of(createColumn("col1")));
+
+ // Scan has base slot + extra subPath slot from VariantSubPathPruning
+ SlotReference scanSlotBase = createMockSlot("col1", "col1",
Collections.emptyList(), true);
+ SlotReference scanSlotHelper = createMockSlot("col1", "col1",
Arrays.asList("a", "b"), true);
+ LogicalOlapScan scan = createMockScan(ImmutableList.of(scanSlotBase,
scanSlotHelper));
+
+ Map<Slot, Slot> replaceMap = scan.constructReplaceMap(mtmv);
+
+ Assertions.assertEquals(1, replaceMap.size());
+ Assertions.assertSame(scanSlotBase, replaceMap.get(mvSlot));
+ }
+
+ /**
+ * Test constructReplaceMap correctly handles aliased columns.
+ * MV SQL: SELECT l_orderkey, sum_total AS agg3, max_total AS agg4 FROM mv1
+ * Plan slots have originalColumn names from source table (sum_total,
max_total),
+ * but MV physical columns are named agg3, agg4 (the aliases).
+ * Physical column name is used as the key, so the mapping succeeds.
+ */
+ @Test
+ public void testConstructReplaceMapWithAliasedColumns() throws Exception {
+ MTMV mtmv = Mockito.mock(MTMV.class);
+ MTMVCache cache = Mockito.mock(MTMVCache.class);
+ Plan originalPlan = Mockito.mock(Plan.class);
+
+ // MV plan output: slot names are from source table.
+ // Use 2-arg createMockSlot (no originalColumn) to avoid nested mock
Column objects
+ // causing Mockito UnfinishedStubbingException. constructReplaceMap
only calls
+ // getOriginalColumn() on scan slots, not on MV plan output slots.
+ SlotReference mvSlot1 = createMockSlot("l_orderkey", true);
+ SlotReference mvSlot2 = createMockSlot("sum_total", true);
+ SlotReference mvSlot3 = createMockSlot("max_total", true);
+
+ Mockito.when(mtmv.getOrGenerateCache(Mockito.any())).thenReturn(cache);
+ Mockito.when(mtmv.getName()).thenReturn("test_alias_mv");
+ Mockito.when(cache.getOriginalFinalPlan()).thenReturn(originalPlan);
+
Mockito.when(originalPlan.getOutput()).thenReturn(ImmutableList.of(mvSlot1,
mvSlot2, mvSlot3));
+
+ // Physical columns have aliased names
+ Mockito.when(mtmv.getBaseSchema()).thenReturn(ImmutableList.of(
+ createColumn("l_orderkey"),
+ createColumn("agg3"), // aliased from sum_total
+ createColumn("agg4") // aliased from max_total
+ ));
+
+ // Scan slots reference MV's physical column names
+ SlotReference scanSlot1 = createMockSlot("l_orderkey", "l_orderkey",
Collections.emptyList(), true);
+ SlotReference scanSlot2 = createMockSlot("agg3", "agg3",
Collections.emptyList(), true);
+ SlotReference scanSlot3 = createMockSlot("agg4", "agg4",
Collections.emptyList(), true);
+ LogicalOlapScan scan = createMockScan(ImmutableList.of(scanSlot1,
scanSlot2, scanSlot3));
+
+ Map<Slot, Slot> replaceMap = scan.constructReplaceMap(mtmv);
+
+ // All 3 should be mapped despite plan slots having different names
than scan slots
+ Assertions.assertEquals(3, replaceMap.size());
+ Assertions.assertSame(scanSlot1, replaceMap.get(mvSlot1));
+ Assertions.assertSame(scanSlot2, replaceMap.get(mvSlot2));
+ Assertions.assertSame(scanSlot3, replaceMap.get(mvSlot3));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
