This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 95f10aa8a15 branch-2.1:[fix](mtmv) fix when compatible fail, will
throw NPE (#49875) (#52273)
95f10aa8a15 is described below
commit 95f10aa8a15625ac0e01ff2925c83597f3ab114c
Author: zhangdong <[email protected]>
AuthorDate: Sat Jun 28 10:54:44 2025 +0800
branch-2.1:[fix](mtmv) fix when compatible fail, will throw NPE (#49875)
(#52273)
pick: https://github.com/apache/doris/pull/49875
---
.../main/java/org/apache/doris/alter/Alter.java | 5 ++
.../main/java/org/apache/doris/catalog/MTMV.java | 17 ++++++
.../java/org/apache/doris/mtmv/BaseTableInfo.java | 16 +++++-
.../org/apache/doris/mtmv/MTMVPartitionInfo.java | 2 +-
.../doris/mtmv/MTMVRefreshPartitionSnapshot.java | 61 +++++++++++-----------
.../org/apache/doris/mtmv/MTMVRefreshSnapshot.java | 2 +-
.../java/org/apache/doris/mtmv/MTMVRelation.java | 4 +-
.../org/apache/doris/mtmv/MTMVRelationManager.java | 43 +++++++--------
.../org/apache/doris/mtmv/MTMVRewriteUtil.java | 5 +-
.../java/org/apache/doris/mtmv/MTMVStatus.java | 6 +++
.../org/apache/doris/nereids/StatementContext.java | 6 +++
.../nereids/rules/analysis/CollectRelation.java | 8 ++-
.../InitConsistentMaterializationContextHook.java | 8 +--
.../mv/InitMaterializationContextHook.java | 7 +--
.../java/org/apache/doris/mtmv/AlterMTMVTest.java | 2 +-
.../org/apache/doris/mtmv/MTMVRewriteUtilTest.java | 12 +++--
.../doris/nereids/memo/StructInfoMapTest.java | 18 +++++++
.../doris/nereids/mv/IdStatisticsMapTest.java | 6 +++
.../doris/nereids/mv/MvTableIdIsLongTest.java | 6 +++
.../nereids/mv/OptimizeGetAvailableMvsTest.java | 20 +++++++
.../org/apache/doris/nereids/util/PlanChecker.java | 11 ++++
.../mv/dml/insert/dml_insert_and_overwrite.groovy | 9 ++--
.../mv/dml/outfile/dml_into_outfile.groovy | 5 +-
23 files changed, 191 insertions(+), 88 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
index 3b5c64b7ff2..95e9d92aa4a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
@@ -1009,6 +1009,11 @@ public class Alter {
case ADD_TASK:
mtmv.addTaskResult(alterMTMV.getTask(),
alterMTMV.getRelation(), alterMTMV.getPartitionSnapshots(),
isReplay);
+ // If it is not a replay thread, it means that the current
service is already a new version
+ // and does not require compatibility
+ if (isReplay) {
+ mtmv.compatible(Env.getCurrentEnv().getCatalogMgr());
+ }
break;
default:
throw new RuntimeException("Unknown type value: " +
alterMTMV.getOpType());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
index ea4e94d5c97..c82c749120b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
@@ -480,6 +480,10 @@ public class MTMV extends OlapTable {
this.refreshSnapshot = refreshSnapshot;
}
+ public boolean canBeCandidate() {
+ return getStatus().canBeCandidate();
+ }
+
public void readMvLock() {
this.mvRwLock.readLock().lock();
}
@@ -556,6 +560,19 @@ public class MTMV extends OlapTable {
* The logic here is to be compatible with older versions by converting ID
to name
*/
public void compatible(CatalogMgr catalogMgr) {
+ try {
+ compatibleInternal(catalogMgr);
+ Env.getCurrentEnv().getMtmvService().unregisterMTMV(this);
+ Env.getCurrentEnv().getMtmvService().registerMTMV(this,
this.getDatabase().getId());
+ } catch (Throwable e) {
+ LOG.warn("MTMV compatible failed, dbName: {}, mvName: {}, errMsg:
{}", getQualifiedDbName(), name,
+ e.getMessage());
+ status.setState(MTMVState.SCHEMA_CHANGE);
+ status.setSchemaChangeDetail("compatible failed, please refresh or
recreate it, reason: " + e.getMessage());
+ }
+ }
+
+ private void compatibleInternal(CatalogMgr catalogMgr) throws Exception {
if (mvPartitionInfo != null) {
mvPartitionInfo.compatible(catalogMgr);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java
index 2f0810c7a72..49f0d4f948b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java
@@ -159,10 +159,18 @@ public class BaseTableInfo {
+ '}';
}
- public void compatible(CatalogMgr catalogMgr) {
+ public void compatible(CatalogMgr catalogMgr) throws Exception {
if (!StringUtils.isEmpty(ctlName)) {
return;
}
+ // should not get meta from external catalog when replay, because the
timeout period may be very long
+ if (ctlId != InternalCatalog.INTERNAL_CATALOG_ID) {
+ String msg = String.format(
+ "Can not compatibility external table, ctlId: %s, dbId:
%s, tableId: %s",
+ ctlId, dbId, tableId);
+ LOG.warn(msg);
+ throw new Exception(msg);
+ }
try {
CatalogIf catalog =
catalogMgr.getCatalogOrAnalysisException(ctlId);
DatabaseIf db = catalog.getDbOrAnalysisException(dbId);
@@ -171,7 +179,11 @@ public class BaseTableInfo {
this.dbName = db.getFullName();
this.tableName = table.getName();
} catch (AnalysisException e) {
- LOG.warn("MTMV compatible failed, ctlId: {}, dbId: {}, tableId:
{}", ctlId, dbId, tableId, e);
+ String msg = String.format(
+ "Failed to get name based on id during compatibility
process, ctlId: %s, dbId: %s, tableId: %s",
+ ctlId, dbId, tableId);
+ LOG.warn(msg, e);
+ throw new Exception(msg);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
index 7eae44db0af..682273a93cd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
@@ -152,7 +152,7 @@ public class MTMVPartitionInfo {
}
}
- public void compatible(CatalogMgr catalogMgr) {
+ public void compatible(CatalogMgr catalogMgr) throws Exception {
if (relatedTable == null) {
return;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
index a8de5b6597b..2601ace5ed6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
@@ -20,8 +20,9 @@ package org.apache.doris.mtmv;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
-import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
@@ -77,29 +78,28 @@ public class MTMVRefreshPartitionSnapshot {
+ '}';
}
- public void compatible(MTMV mtmv) {
- try {
- // snapshot add partitionId resolve problem of insert overwrite
- compatiblePartitions(mtmv);
- } catch (Throwable e) {
- LOG.warn("MTMV compatiblePartitions failed, mtmv: {}",
mtmv.getName(), e);
- }
- try {
- // change table id to BaseTableInfo
- compatibleTables(mtmv);
- } catch (Throwable e) {
- LOG.warn("MTMV compatibleTables failed, mtmv: {}", mtmv.getName(),
e);
- }
-
- try {
- // snapshot add tableId resolve problem of recreate table
- compatibleTablesSnapshot();
- } catch (Throwable e) {
- LOG.warn("MTMV compatibleTables failed, mtmv: {}", mtmv.getName(),
e);
- }
+ public void compatible(MTMV mtmv) throws Exception {
+ // snapshot add partitionId resolve problem of insert overwrite
+ compatiblePartitions(mtmv);
+ // change table id to BaseTableInfo
+ compatibleTables(mtmv);
+ // snapshot add tableId resolve problem of recreate table
+ compatibleTablesSnapshot();
}
private void compatiblePartitions(MTMV mtmv) throws AnalysisException {
+ if
(mtmv.getMvPartitionInfo().getPartitionType().equals(MTMVPartitionType.SELF_MANAGE))
{
+ return;
+ }
+ // Only olapTable has historical data issues that require compatibility
+ if (mtmv.getMvPartitionInfo().getRelatedTableInfo().getCtlId() !=
InternalCatalog.INTERNAL_CATALOG_ID) {
+ return;
+ }
+ MTMVRelatedTableIf relatedTableIf =
mtmv.getMvPartitionInfo().getRelatedTable();
+ // Only olapTable has historical data issues that require compatibility
+ if (!(relatedTableIf instanceof OlapTable)) {
+ return;
+ }
if (!checkHasDataWithoutPartitionId()) {
return;
}
@@ -108,6 +108,8 @@ public class MTMVRefreshPartitionSnapshot {
MTMVVersionSnapshot versionSnapshot = (MTMVVersionSnapshot)
entry.getValue();
if (versionSnapshot.getId() == 0) {
Partition partition =
relatedTable.getPartition(entry.getKey());
+ // if not find partition, may be partition has been dropped,
+ // the impact is that MTMV will consider this partition to be
async
if (partition != null) {
(versionSnapshot).setId(partition.getId());
}
@@ -131,12 +133,7 @@ public class MTMVRefreshPartitionSnapshot {
for (Entry<BaseTableInfo, MTMVSnapshotIf> entry :
tablesInfo.entrySet()) {
MTMVVersionSnapshot versionSnapshot = (MTMVVersionSnapshot)
entry.getValue();
if (versionSnapshot.getId() == 0) {
- try {
- TableIf table = MTMVUtil.getTable(entry.getKey());
- versionSnapshot.setId(table.getId());
- } catch (AnalysisException e) {
- LOG.warn("MTMV compatibleTablesSnapshot failed, can not
get table by: {}", entry.getKey());
- }
+ versionSnapshot.setId(entry.getKey().getTableId());
}
}
}
@@ -150,7 +147,7 @@ public class MTMVRefreshPartitionSnapshot {
return false;
}
- private void compatibleTables(MTMV mtmv) {
+ private void compatibleTables(MTMV mtmv) throws Exception {
if (tables.size() == tablesInfo.size()) {
return;
}
@@ -164,8 +161,12 @@ public class MTMVRefreshPartitionSnapshot {
if (tableInfo.isPresent()) {
tablesInfo.put(tableInfo.get(), entry.getValue());
} else {
- LOG.warn("MTMV compatibleTables failed, tableId: {},
relationTables: {}", entry.getKey(),
- relation.getBaseTablesOneLevel());
+ String msg = String.format(
+ "Failed to get table info based on id during
compatibility process, "
+ + "tableId: %s, relationTables: %s",
+ entry.getKey(), relation.getBaseTablesOneLevel());
+ LOG.warn(msg);
+ throw new Exception(msg);
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java
index 74fc3cc1c5c..0d9665cb446 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java
@@ -91,7 +91,7 @@ public class MTMVRefreshSnapshot {
+ '}';
}
- public void compatible(MTMV mtmv) {
+ public void compatible(MTMV mtmv) throws Exception {
if (MapUtils.isEmpty(partitionSnapshots)) {
return;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java
index 87a0199f128..148d2d00884 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java
@@ -65,13 +65,13 @@ public class MTMVRelation {
+ '}';
}
- public void compatible(CatalogMgr catalogMgr) {
+ public void compatible(CatalogMgr catalogMgr) throws Exception {
compatible(catalogMgr, baseTables);
compatible(catalogMgr, baseViews);
compatible(catalogMgr, baseTablesOneLevel);
}
- private void compatible(CatalogMgr catalogMgr, Set<BaseTableInfo> infos) {
+ private void compatible(CatalogMgr catalogMgr, Set<BaseTableInfo> infos)
throws Exception {
if (CollectionUtils.isEmpty(infos)) {
return;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
index c45558ec8cf..ba498636d73 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
@@ -78,48 +78,43 @@ public class MTMVRelationManager implements MTMVHookService
{
/**
* if At least one partition is available, return this mtmv
*
- * @param tableInfos
+ * @param candidateMTMVs
* @param ctx
* @return
*/
- public Set<MTMV> getAvailableMTMVs(List<BaseTableInfo> tableInfos,
ConnectContext ctx,
+ public Set<MTMV> getAvailableMTMVs(Set<MTMV> candidateMTMVs,
ConnectContext ctx,
boolean forceConsistent, BiPredicate<ConnectContext, MTMV>
predicate) {
Set<MTMV> res = Sets.newLinkedHashSet();
- Set<BaseTableInfo> mvInfos = getMTMVInfos(tableInfos);
Map<List<String>, Set<String>> queryUsedPartitions =
PartitionCompensator.getQueryUsedPartitions(
ctx.getStatementContext(), new BitSet());
-
- for (BaseTableInfo tableInfo : mvInfos) {
- try {
- MTMV mtmv = (MTMV) MTMVUtil.getTable(tableInfo);
- if (predicate.test(ctx, mtmv)) {
- continue;
- }
- if (!mtmv.isUseForRewrite()) {
- continue;
- }
- BaseTableInfo relatedTableInfo =
mtmv.getMvPartitionInfo().getRelatedTableInfo();
- if (isMVPartitionValid(mtmv, ctx, forceConsistent,
- relatedTableInfo == null ? null :
queryUsedPartitions.get(relatedTableInfo.toList()))) {
- res.add(mtmv);
- }
- } catch (Exception e) {
- // not throw exception to client, just ignore it
- LOG.warn("getTable failed: {}", tableInfo.toString(), e);
+ for (MTMV mtmv : candidateMTMVs) {
+ if (predicate.test(ctx, mtmv)) {
+ continue;
+ }
+ if (!mtmv.isUseForRewrite()) {
+ continue;
+ }
+ BaseTableInfo relatedTableInfo =
mtmv.getMvPartitionInfo().getRelatedTableInfo();
+ if (isMVPartitionValid(mtmv, ctx, forceConsistent,
+ relatedTableInfo == null ? null :
queryUsedPartitions.get(relatedTableInfo.toList()))) {
+ res.add(mtmv);
}
}
return res;
}
/**
- * get all mtmv related to tableInfos.
+ * get candidate mtmv related to tableInfos.
*/
- public Set<MTMV> getAllMTMVs(List<BaseTableInfo> tableInfos) {
+ public Set<MTMV> getCandidateMTMVs(List<BaseTableInfo> tableInfos) {
Set<MTMV> mtmvs = Sets.newLinkedHashSet();
Set<BaseTableInfo> mvInfos = getMTMVInfos(tableInfos);
for (BaseTableInfo tableInfo : mvInfos) {
try {
- mtmvs.add((MTMV) MTMVUtil.getTable(tableInfo));
+ MTMV mtmv = (MTMV) MTMVUtil.getTable(tableInfo);
+ if (mtmv.canBeCandidate()) {
+ mtmvs.add(mtmv);
+ }
} catch (Exception e) {
// not throw exception to client, just ignore it
LOG.warn("getTable failed: {}", tableInfo.toString(), e);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
index afaad55a34b..58b2a37d504 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
@@ -20,8 +20,6 @@ package org.apache.doris.mtmv;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.Partition;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVRefreshState;
-import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Lists;
@@ -56,8 +54,7 @@ public class MTMVRewriteUtil {
return res;
}
// check mv is normal
- MTMVStatus mtmvStatus = mtmv.getStatus();
- if (mtmvStatus.getState() != MTMVState.NORMAL ||
mtmvStatus.getRefreshState() == MTMVRefreshState.INIT) {
+ if (!mtmv.canBeCandidate()) {
return res;
}
// if relatedPartitions is empty but not null, which means query no
partitions
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java
index b1761b9e973..aa058e628c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java
@@ -82,6 +82,12 @@ public class MTMVStatus {
return this;
}
+ public boolean canBeCandidate() {
+ // MTMVRefreshState.FAIL also can be candidate, because may have some
sync partitions
+ return getState() == MTMVState.NORMAL
+ && getRefreshState() != MTMVRefreshState.INIT;
+ }
+
@Override
public String toString() {
return "MTMVStatus{"
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 009c51595b9..cc04721ab80 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.View;
@@ -189,6 +190,7 @@ public class StatementContext implements Closeable {
// if query is: select * from t2 join t5
// mtmvRelatedTables is mv1, mv2, mv3, t1, t2, t3, t4, t5
private final Map<List<String>, TableIf> mtmvRelatedTables =
Maps.newHashMap();
+ private final Set<MTMV> candidateMTMVs = Sets.newHashSet();
// insert into target tables
private final Map<List<String>, TableIf> insertTargetTables =
Maps.newHashMap();
// save view's def to avoid them change before lock
@@ -306,6 +308,10 @@ public class StatementContext implements Closeable {
return mtmvRelatedTables;
}
+ public Set<MTMV> getCandidateMTMVs() {
+ return candidateMTMVs;
+ }
+
public Map<List<String>, TableIf> getTables() {
return tables;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
index 410028ff735..c529cfb580b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
@@ -207,9 +207,12 @@ public class CollectRelation implements
AnalysisRuleFactory {
}
if (shouldCollect) {
Set<MTMV> mtmvSet =
Env.getCurrentEnv().getMtmvService().getRelationManager()
- .getAllMTMVs(Lists.newArrayList(new BaseTableInfo(table)));
- LOG.info("table {} related mv set is {}", new
BaseTableInfo(table), mtmvSet);
+ .getCandidateMTMVs(Lists.newArrayList(new
BaseTableInfo(table)));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("table {} related mv set is {}", new
BaseTableInfo(table), mtmvSet);
+ }
for (MTMV mtmv : mtmvSet) {
+
cascadesContext.getStatementContext().getCandidateMTMVs().add(mtmv);
cascadesContext.getStatementContext().getMtmvRelatedTables().put(mtmv.getFullQualifiers(),
mtmv);
mtmv.readMvLock();
try {
@@ -221,6 +224,7 @@ public class CollectRelation implements AnalysisRuleFactory
{
LOG.debug("mtmv {} related base table include {}",
new BaseTableInfo(mtmv), baseTableInfo);
}
try {
+ // Collect all base tables and lock them before
querying
cascadesContext.getStatementContext().getAndCacheTable(baseTableInfo.toList(),
TableFrom.MTMV);
} catch (AnalysisException exception) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java
index fbcf4726a10..e86cca263d3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java
@@ -20,16 +20,13 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.TableIf;
-import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVUtil;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.PlannerHook;
import com.google.common.annotations.VisibleForTesting;
-import java.util.List;
import java.util.Set;
-import java.util.stream.Collectors;
/**
* If enable query rewrite with mv in dml, should init consistent
materialization context after analyze
@@ -49,10 +46,9 @@ public class InitConsistentMaterializationContextHook
extends InitMaterializatio
}
protected Set<MTMV> getAvailableMTMVs(Set<TableIf> usedTables,
CascadesContext cascadesContext) {
- List<BaseTableInfo> usedBaseTables =
-
usedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList());
return Env.getCurrentEnv().getMtmvService().getRelationManager()
- .getAvailableMTMVs(usedBaseTables,
cascadesContext.getConnectContext(),
+
.getAvailableMTMVs(cascadesContext.getStatementContext().getCandidateMTMVs(),
+ cascadesContext.getConnectContext(),
true, ((connectContext, mtmv) -> {
return MTMVUtil.mtmvContainsExternalTable(mtmv) &&
(!connectContext.getSessionVariable()
.isEnableDmlMaterializedViewRewriteWhenBaseTableUnawareness());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
index 914ae3a5195..ba3afad0fe6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
@@ -20,7 +20,6 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.TableIf;
-import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVCache;
import org.apache.doris.mtmv.MTMVUtil;
import org.apache.doris.nereids.CascadesContext;
@@ -39,7 +38,6 @@ import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
import java.util.Set;
-import java.util.stream.Collectors;
/**
* If enable query rewrite with mv, should init materialization context after
analyze
@@ -93,10 +91,9 @@ public class InitMaterializationContextHook implements
PlannerHook {
}
protected Set<MTMV> getAvailableMTMVs(Set<TableIf> usedTables,
CascadesContext cascadesContext) {
- List<BaseTableInfo> usedBaseTables =
-
usedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList());
return Env.getCurrentEnv().getMtmvService().getRelationManager()
- .getAvailableMTMVs(usedBaseTables,
cascadesContext.getConnectContext(),
+
.getAvailableMTMVs(cascadesContext.getStatementContext().getCandidateMTMVs(),
+ cascadesContext.getConnectContext(),
false, ((connectContext, mtmv) -> {
return MTMVUtil.mtmvContainsExternalTable(mtmv) &&
(!connectContext.getSessionVariable()
.isEnableMaterializedViewRewriteWhenBaseTableUnawareness());
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java
index 17ec145f583..342f9fd60c8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java
@@ -51,7 +51,7 @@ public class AlterMTMVTest extends TestWithFeService {
MTMVRelationManager relationManager =
Env.getCurrentEnv().getMtmvService().getRelationManager();
Table table =
Env.getCurrentInternalCatalog().getDb("test").get().getTableOrMetaException("stu");
- Set<MTMV> allMTMVs =
relationManager.getAllMTMVs(Lists.newArrayList(new BaseTableInfo(table)));
+ Set<MTMV> allMTMVs =
relationManager.getCandidateMTMVs(Lists.newArrayList(new BaseTableInfo(table)));
boolean hasMvA = false;
boolean hasMvB = false;
for (MTMV mtmv : allMTMVs) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
index 82c7eaac631..e4788c18409 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
@@ -122,6 +122,10 @@ public class MTMVRewriteUtilTest {
MTMVUtil.mtmvContainsExternalTable((MTMV) any);
minTimes = 0;
result = false;
+
+ mtmv.canBeCandidate();
+ minTimes = 0;
+ result = true;
}
};
}
@@ -279,9 +283,9 @@ public class MTMVRewriteUtilTest {
public void testGetMTMVCanRewritePartitionsStateAbnormal() {
new Expectations() {
{
- status.getState();
+ mtmv.canBeCandidate();
minTimes = 0;
- result = MTMVState.SCHEMA_CHANGE;
+ result = false;
}
};
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
@@ -309,9 +313,9 @@ public class MTMVRewriteUtilTest {
public void testGetMTMVCanRewritePartitionsRefreshStateInit() {
new Expectations() {
{
- status.getRefreshState();
+ mtmv.canBeCandidate();
minTimes = 0;
- result = MTMVRefreshState.INIT;
+ result = false;
}
};
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
index 19d1efdbbd8..0d3181d15e8 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
@@ -70,6 +70,12 @@ class StructInfoMapTest extends SqlTestBase {
return true;
}
};
+ new MockUp<MTMV>() {
+ @Mock
+ public boolean canBeCandidate() {
+ return true;
+ }
+ };
connectContext.getSessionVariable().enableMaterializedViewRewrite =
true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite
= true;
@@ -129,6 +135,12 @@ class StructInfoMapTest extends SqlTestBase {
return true;
}
};
+ new MockUp<MTMV>() {
+ @Mock
+ public boolean canBeCandidate() {
+ return true;
+ }
+ };
connectContext.getSessionVariable().enableMaterializedViewRewrite =
true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite
= true;
createMvByNereids("create materialized view mv1 BUILD IMMEDIATE
REFRESH COMPLETE ON MANUAL\n"
@@ -177,6 +189,12 @@ class StructInfoMapTest extends SqlTestBase {
return true;
}
};
+ new MockUp<MTMV>() {
+ @Mock
+ public boolean canBeCandidate() {
+ return true;
+ }
+ };
connectContext.getSessionVariable().enableMaterializedViewRewrite =
true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite
= true;
createMvByNereids("create materialized view mv1 BUILD IMMEDIATE
REFRESH COMPLETE ON MANUAL\n"
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java
index 1403a9fee5e..3e93919ed22 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java
@@ -59,6 +59,12 @@ public class IdStatisticsMapTest extends SqlTestBase {
return true;
}
};
+ new MockUp<MTMV>() {
+ @Mock
+ public boolean canBeCandidate() {
+ return true;
+ }
+ };
connectContext.getSessionVariable().enableMaterializedViewRewrite =
true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite
= true;
createMvByNereids("create materialized view mv100 BUILD IMMEDIATE
REFRESH COMPLETE ON MANUAL\n"
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java
index 4fa0a68e77c..5c9fb3c0392 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java
@@ -55,6 +55,12 @@ public class MvTableIdIsLongTest extends SqlTestBase {
return true;
}
};
+ new MockUp<MTMV>() {
+ @Mock
+ public boolean canBeCandidate() {
+ return true;
+ }
+ };
connectContext.getSessionVariable().enableMaterializedViewRewrite =
true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite
= true;
createMvByNereids("create materialized view mv1 BUILD IMMEDIATE
REFRESH COMPLETE ON MANUAL\n"
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/OptimizeGetAvailableMvsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/OptimizeGetAvailableMvsTest.java
index da327b2ba6c..ab70b9dac4a 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/OptimizeGetAvailableMvsTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/OptimizeGetAvailableMvsTest.java
@@ -18,13 +18,16 @@
package org.apache.doris.nereids.mv;
import org.apache.doris.catalog.DistributionInfo;
+import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.MaterializedIndex.IndexState;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.common.Pair;
+import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.mtmv.BaseTableInfo;
+import org.apache.doris.mtmv.MTMVRelationManager;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.expression.rules.PartitionPruner;
import
org.apache.doris.nereids.rules.expression.rules.PartitionPruner.PartitionTableType;
@@ -48,6 +51,7 @@ import java.util.BitSet;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
/**
@@ -127,6 +131,14 @@ public class OptimizeGetAvailableMvsTest extends
SqlTestBase {
+ "inner join T3 on T4.id = T3.id",
connectContext
);
+ CatalogIf internal = getCatalog("internal");
+ Optional table =
internal.getDbOrAnalysisException("test").getTable("mv1");
+ new MockUp<MTMVRelationManager>() {
+ @Mock
+ public Set<MTMV> getCandidateMTMVs(List<BaseTableInfo>
baseTableInfos) {
+ return Sets.newHashSet((MTMV) table.get());
+ }
+ };
PlanChecker.from(c1)
.analyze()
.rewrite()
@@ -238,6 +250,14 @@ public class OptimizeGetAvailableMvsTest extends
SqlTestBase {
+ "where T4.id > 0",
connectContext
);
+ CatalogIf internal = getCatalog("internal");
+ Optional table =
internal.getDbOrAnalysisException("test").getTable("mv2");
+ new MockUp<MTMVRelationManager>() {
+ @Mock
+ public Set<MTMV> getCandidateMTMVs(List<BaseTableInfo>
baseTableInfos) {
+ return Sets.newHashSet((MTMV) table.get());
+ }
+ };
PlanChecker.from(c1)
.analyze()
.rewrite()
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
index e535e3a8ac5..71d0f0101b0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.util;
import org.apache.doris.analysis.ExplainOptions;
+import org.apache.doris.nereids.CTEContext;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.PlanProcess;
@@ -49,6 +50,7 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleFactory;
import org.apache.doris.nereids.rules.RuleSet;
import org.apache.doris.nereids.rules.RuleType;
+import
org.apache.doris.nereids.rules.exploration.mv.InitConsistentMaterializationContextHook;
import
org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
@@ -120,6 +122,9 @@ public class PlanChecker {
}
public PlanChecker analyze() {
+
this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE);
+ this.cascadesContext.newTableCollector().collect();
+ this.cascadesContext.setCteContext(new CTEContext());
this.cascadesContext.newAnalyzer().analyze();
this.cascadesContext.toMemo();
return this;
@@ -127,6 +132,9 @@ public class PlanChecker {
public PlanChecker analyze(Plan plan) {
this.cascadesContext =
MemoTestUtils.createCascadesContext(connectContext, plan);
+
this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE);
+ this.cascadesContext.newTableCollector().collect();
+ this.cascadesContext.setCteContext(new CTEContext());
Set<String> originDisableRules =
connectContext.getSessionVariable().getDisableNereidsRuleNames();
Set<String> disableRuleWithAuth = Sets.newHashSet(originDisableRules);
disableRuleWithAuth.add(RuleType.RELATION_AUTHENTICATION.name());
@@ -140,6 +148,9 @@ public class PlanChecker {
public PlanChecker analyze(String sql) {
this.cascadesContext =
MemoTestUtils.createCascadesContext(connectContext, sql);
+
this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE);
+ this.cascadesContext.newTableCollector().collect();
+ this.cascadesContext.setCteContext(new CTEContext());
this.cascadesContext.newAnalyzer().analyze();
this.cascadesContext.toMemo();
return this;
diff --git
a/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy
b/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy
index 54e9b57f7e5..753217f922d 100644
---
a/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy
@@ -106,7 +106,8 @@ suite("dml_insert_and_overwrite") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ // todo: Temporarily turn off, otherwise usable materialized views will
not be collected and will need to be changed back in the future
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
@@ -155,7 +156,7 @@ suite("dml_insert_and_overwrite") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
@@ -203,7 +204,7 @@ suite("dml_insert_and_overwrite") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
@@ -249,7 +250,7 @@ suite("dml_insert_and_overwrite") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
diff --git
a/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy
b/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy
index 350e49057d0..dc23130b990 100644
---
a/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy
@@ -122,7 +122,8 @@ suite("dml_into_outfile", "p0") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ // todo: Temporarily turn off, otherwise usable materialized views will
not be collected and will need to be changed back in the future
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
@@ -185,7 +186,7 @@ suite("dml_into_outfile", "p0") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
