This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4708d9cad6b [fix](mtmv) fix when compatible fail, will throw NPE
(#49875)
4708d9cad6b is described below
commit 4708d9cad6bc088bbe4df688408cdd79821945b2
Author: zhangdong <[email protected]>
AuthorDate: Tue Jun 24 10:57:57 2025 +0800
[fix](mtmv) fix when compatible fail, will throw NPE (#49875)
### What problem does this PR solve?
The old version of the materialized view stores the ID of the
relationship between the base table, while the new version stores the
name. During the upgrade process, the name will be checked for
compatibility with the old version based on the ID. However, when the
table is deleted, the name will be empty
so, when get table by empty name , while throw NPE
now, when table is deleted,set status of mtmv to SCHEMA_CHANGE, and when
query, not use mtmv with status SCHEMA_CHANGE
- MTMV adds the canBeCandidate method to determine whether the current
state of the materialized view can be transparently rewritten
- org. apache.doris.nereids. rules. analysis. CollectRelation #
collectMTMVCandidates method replaces getAllMTMVs with getCandidateMTMVs
and stores the result in StatementContext,During the transparent
rewriting process, the results are directly obtained from the
statementContext to ensure consistency in the number of materialized
views throughout the entire process
- During the process of replaying the image and editLog, metadata
compatibility is also performed to avoid inconsistencies in multiple
node versions. However, exceptions should be caught because the metadata
involved in the compatibility process may not be ready yet
---
.../main/java/org/apache/doris/alter/Alter.java | 5 ++
.../main/java/org/apache/doris/catalog/Env.java | 6 ++-
.../main/java/org/apache/doris/catalog/MTMV.java | 16 +++++++
.../java/org/apache/doris/mtmv/BaseTableInfo.java | 16 ++++++-
.../org/apache/doris/mtmv/MTMVPartitionInfo.java | 2 +-
.../doris/mtmv/MTMVRefreshPartitionSnapshot.java | 56 ++++++++++------------
.../org/apache/doris/mtmv/MTMVRefreshSnapshot.java | 2 +-
.../java/org/apache/doris/mtmv/MTMVRelation.java | 4 +-
.../org/apache/doris/mtmv/MTMVRelationManager.java | 43 ++++++++---------
.../org/apache/doris/mtmv/MTMVRewriteUtil.java | 5 +-
.../java/org/apache/doris/mtmv/MTMVStatus.java | 6 +++
.../org/apache/doris/nereids/StatementContext.java | 6 +++
.../nereids/rules/analysis/CollectRelation.java | 4 +-
.../InitConsistentMaterializationContextHook.java | 8 +---
.../mv/InitMaterializationContextHook.java | 7 +--
.../java/org/apache/doris/mtmv/AlterMTMVTest.java | 2 +-
.../org/apache/doris/mtmv/MTMVRewriteUtilTest.java | 12 +++--
.../doris/nereids/memo/StructInfoMapTest.java | 18 +++++++
.../doris/nereids/mv/IdStatisticsMapTest.java | 6 +++
.../doris/nereids/mv/MvTableIdIsLongTest.java | 6 +++
.../org/apache/doris/nereids/util/PlanChecker.java | 11 +++++
.../suites/mtmv_p0/test_paimon_mtmv.groovy | 4 +-
.../mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy | 4 +-
.../suites/mtmv_p0/test_paimon_rewrite_mtmv.groovy | 4 +-
.../mv/dml/insert/dml_insert_and_overwrite.groovy | 9 ++--
.../mv/dml/outfile/dml_into_outfile.groovy | 5 +-
26 files changed, 176 insertions(+), 91 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
index 079db77d24b..a5f541db9ef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
@@ -1299,6 +1299,11 @@ public class Alter {
case ADD_TASK:
mtmv.addTaskResult(alterMTMV.getTask(),
alterMTMV.getRelation(), alterMTMV.getPartitionSnapshots(),
isReplay);
+ // If it is not a replay thread, it means that the current
service is already a new version
+ // and does not require compatibility
+ if (isReplay) {
+ mtmv.compatible(Env.getCurrentEnv().getCatalogMgr());
+ }
break;
default:
throw new RuntimeException("Unknown type value: " +
alterMTMV.getOpType());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 605ed7ddd2d..0486af68d4c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -4317,7 +4317,11 @@ public class Env {
public void replayCreateTable(CreateTableInfo info) throws
MetaNotFoundException {
if (Strings.isNullOrEmpty(info.getCtlName()) || info.getCtlName()
.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
- getInternalCatalog().replayCreateTable(info.getDbName(),
info.getTable());
+ Table table = info.getTable();
+ getInternalCatalog().replayCreateTable(info.getDbName(), table);
+ if (table instanceof MTMV) {
+ ((MTMV) table).compatible(Env.getCurrentEnv().getCatalogMgr());
+ }
} else {
ExternalCatalog externalCatalog = (ExternalCatalog)
catalogMgr.getCatalog(info.getCtlName());
if (externalCatalog != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
index ebd928c867c..7637b569fcc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
@@ -474,6 +474,10 @@ public class MTMV extends OlapTable {
this.refreshSnapshot = refreshSnapshot;
}
+ public boolean canBeCandidate() {
+ return getStatus().canBeCandidate();
+ }
+
public void readMvLock() {
this.mvRwLock.readLock().lock();
}
@@ -521,6 +525,18 @@ public class MTMV extends OlapTable {
* The logic here is to be compatible with older versions by converting ID
to name
*/
public void compatible(CatalogMgr catalogMgr) {
+ try {
+ compatibleInternal(catalogMgr);
+ Env.getCurrentEnv().getMtmvService().unregisterMTMV(this);
+ Env.getCurrentEnv().getMtmvService().registerMTMV(this,
this.getDatabase().getId());
+ } catch (Throwable e) {
+ LOG.warn("MTMV compatible failed, dbName: {}, mvName: {}, errMsg:
{}", getDBName(), name, e.getMessage());
+ status.setState(MTMVState.SCHEMA_CHANGE);
+ status.setSchemaChangeDetail("compatible failed, please refresh or
recreate it, reason: " + e.getMessage());
+ }
+ }
+
+ private void compatibleInternal(CatalogMgr catalogMgr) throws Exception {
if (mvPartitionInfo != null) {
mvPartitionInfo.compatible(catalogMgr);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java
index 625fe7d9187..b2b20ec0bab 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java
@@ -159,10 +159,18 @@ public class BaseTableInfo {
+ '}';
}
- public void compatible(CatalogMgr catalogMgr) {
+ public void compatible(CatalogMgr catalogMgr) throws Exception {
if (!StringUtils.isEmpty(ctlName)) {
return;
}
+ // should not get meta from external catalog when replay, because the
timeout period may be very long
+ if (ctlId != InternalCatalog.INTERNAL_CATALOG_ID) {
+ String msg = String.format(
+ "Can not compatibility external table, ctlId: %s, dbId:
%s, tableId: %s",
+ ctlId, dbId, tableId);
+ LOG.warn(msg);
+ throw new Exception(msg);
+ }
try {
CatalogIf catalog =
catalogMgr.getCatalogOrAnalysisException(ctlId);
DatabaseIf db = catalog.getDbOrAnalysisException(dbId);
@@ -171,7 +179,11 @@ public class BaseTableInfo {
this.dbName = db.getFullName();
this.tableName = table.getName();
} catch (AnalysisException e) {
- LOG.warn("MTMV compatible failed, ctlId: {}, dbId: {}, tableId:
{}", ctlId, dbId, tableId, e);
+ String msg = String.format(
+ "Failed to get name based on id during compatibility
process, ctlId: %s, dbId: %s, tableId: %s",
+ ctlId, dbId, tableId);
+ LOG.warn(msg, e);
+ throw new Exception(msg);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
index 5a14867c7e1..d34580f6608 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
@@ -154,7 +154,7 @@ public class MTMVPartitionInfo {
}
}
- public void compatible(CatalogMgr catalogMgr) {
+ public void compatible(CatalogMgr catalogMgr) throws Exception {
if (relatedTable == null) {
return;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
index d7d9b65f4a5..490fc8ca2fe 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
@@ -20,8 +20,9 @@ package org.apache.doris.mtmv;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
-import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
@@ -77,29 +78,23 @@ public class MTMVRefreshPartitionSnapshot {
+ '}';
}
- public void compatible(MTMV mtmv) {
- try {
- // snapshot add partitionId resolve problem of insert overwrite
- compatiblePartitions(mtmv);
- } catch (Throwable e) {
- LOG.warn("MTMV compatiblePartitions failed, mtmv: {}",
mtmv.getName(), e);
- }
- try {
- // change table id to BaseTableInfo
- compatibleTables(mtmv);
- } catch (Throwable e) {
- LOG.warn("MTMV compatibleTables failed, mtmv: {}", mtmv.getName(),
e);
- }
-
- try {
- // snapshot add tableId resolve problem of recreate table
- compatibleTablesSnapshot();
- } catch (Throwable e) {
- LOG.warn("MTMV compatibleTables failed, mtmv: {}", mtmv.getName(),
e);
- }
+ public void compatible(MTMV mtmv) throws Exception {
+ // snapshot add partitionId resolve problem of insert overwrite
+ compatiblePartitions(mtmv);
+ // change table id to BaseTableInfo
+ compatibleTables(mtmv);
+ // snapshot add tableId resolve problem of recreate table
+ compatibleTablesSnapshot();
}
private void compatiblePartitions(MTMV mtmv) throws AnalysisException {
+ if
(mtmv.getMvPartitionInfo().getPartitionType().equals(MTMVPartitionType.SELF_MANAGE))
{
+ return;
+ }
+ // Only olapTable has historical data issues that require compatibility
+ if (mtmv.getMvPartitionInfo().getRelatedTableInfo().getCtlId() !=
InternalCatalog.INTERNAL_CATALOG_ID) {
+ return;
+ }
MTMVRelatedTableIf relatedTableIf =
mtmv.getMvPartitionInfo().getRelatedTable();
// Only olapTable has historical data issues that require compatibility
if (!(relatedTableIf instanceof OlapTable)) {
@@ -113,6 +108,8 @@ public class MTMVRefreshPartitionSnapshot {
MTMVVersionSnapshot versionSnapshot = (MTMVVersionSnapshot)
entry.getValue();
if (versionSnapshot.getId() == 0) {
Partition partition =
relatedTable.getPartition(entry.getKey());
+ // if not find partition, may be partition has been dropped,
+ // the impact is that MTMV will consider this partition to be
async
if (partition != null) {
(versionSnapshot).setId(partition.getId());
}
@@ -136,12 +133,7 @@ public class MTMVRefreshPartitionSnapshot {
for (Entry<BaseTableInfo, MTMVSnapshotIf> entry :
tablesInfo.entrySet()) {
MTMVVersionSnapshot versionSnapshot = (MTMVVersionSnapshot)
entry.getValue();
if (versionSnapshot.getId() == 0) {
- try {
- TableIf table = MTMVUtil.getTable(entry.getKey());
- versionSnapshot.setId(table.getId());
- } catch (AnalysisException e) {
- LOG.warn("MTMV compatibleTablesSnapshot failed, can not
get table by: {}", entry.getKey());
- }
+ versionSnapshot.setId(entry.getKey().getTableId());
}
}
}
@@ -155,7 +147,7 @@ public class MTMVRefreshPartitionSnapshot {
return false;
}
- private void compatibleTables(MTMV mtmv) {
+ private void compatibleTables(MTMV mtmv) throws Exception {
if (tables.size() == tablesInfo.size()) {
return;
}
@@ -169,8 +161,12 @@ public class MTMVRefreshPartitionSnapshot {
if (tableInfo.isPresent()) {
tablesInfo.put(tableInfo.get(), entry.getValue());
} else {
- LOG.warn("MTMV compatibleTables failed, tableId: {},
relationTables: {}", entry.getKey(),
- relation.getBaseTablesOneLevel());
+ String msg = String.format(
+ "Failed to get table info based on id during
compatibility process, "
+ + "tableId: %s, relationTables: %s",
+ entry.getKey(), relation.getBaseTablesOneLevel());
+ LOG.warn(msg);
+ throw new Exception(msg);
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java
index 74fc3cc1c5c..0d9665cb446 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java
@@ -91,7 +91,7 @@ public class MTMVRefreshSnapshot {
+ '}';
}
- public void compatible(MTMV mtmv) {
+ public void compatible(MTMV mtmv) throws Exception {
if (MapUtils.isEmpty(partitionSnapshots)) {
return;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java
index 87a0199f128..148d2d00884 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java
@@ -65,13 +65,13 @@ public class MTMVRelation {
+ '}';
}
- public void compatible(CatalogMgr catalogMgr) {
+ public void compatible(CatalogMgr catalogMgr) throws Exception {
compatible(catalogMgr, baseTables);
compatible(catalogMgr, baseViews);
compatible(catalogMgr, baseTablesOneLevel);
}
- private void compatible(CatalogMgr catalogMgr, Set<BaseTableInfo> infos) {
+ private void compatible(CatalogMgr catalogMgr, Set<BaseTableInfo> infos)
throws Exception {
if (CollectionUtils.isEmpty(infos)) {
return;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
index 54db92b7d97..f87bbe4f711 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
@@ -77,48 +77,43 @@ public class MTMVRelationManager implements MTMVHookService
{
/**
* if At least one partition is available, return this mtmv
*
- * @param tableInfos
+ * @param candidateMTMVs
* @param ctx
* @return
*/
- public Set<MTMV> getAvailableMTMVs(List<BaseTableInfo> tableInfos,
ConnectContext ctx,
+ public Set<MTMV> getAvailableMTMVs(Set<MTMV> candidateMTMVs,
ConnectContext ctx,
boolean forceConsistent, BiPredicate<ConnectContext, MTMV>
predicate) {
Set<MTMV> res = Sets.newLinkedHashSet();
- Set<BaseTableInfo> mvInfos = getMTMVInfos(tableInfos);
Map<List<String>, Set<String>> queryUsedPartitions =
PartitionCompensator.getQueryUsedPartitions(
ctx.getStatementContext(), new BitSet());
-
- for (BaseTableInfo tableInfo : mvInfos) {
- try {
- MTMV mtmv = (MTMV) MTMVUtil.getTable(tableInfo);
- if (predicate.test(ctx, mtmv)) {
- continue;
- }
- if (!mtmv.isUseForRewrite()) {
- continue;
- }
- BaseTableInfo relatedTableInfo =
mtmv.getMvPartitionInfo().getRelatedTableInfo();
- if (isMVPartitionValid(mtmv, ctx, forceConsistent,
- relatedTableInfo == null ? null :
queryUsedPartitions.get(relatedTableInfo.toList()))) {
- res.add(mtmv);
- }
- } catch (Exception e) {
- // not throw exception to client, just ignore it
- LOG.warn("getTable failed: {}", tableInfo.toString(), e);
+ for (MTMV mtmv : candidateMTMVs) {
+ if (predicate.test(ctx, mtmv)) {
+ continue;
+ }
+ if (!mtmv.isUseForRewrite()) {
+ continue;
+ }
+ BaseTableInfo relatedTableInfo =
mtmv.getMvPartitionInfo().getRelatedTableInfo();
+ if (isMVPartitionValid(mtmv, ctx, forceConsistent,
+ relatedTableInfo == null ? null :
queryUsedPartitions.get(relatedTableInfo.toList()))) {
+ res.add(mtmv);
}
}
return res;
}
/**
- * get all mtmv related to tableInfos.
+ * get candidate mtmv related to tableInfos.
*/
- public Set<MTMV> getAllMTMVs(List<BaseTableInfo> tableInfos) {
+ public Set<MTMV> getCandidateMTMVs(List<BaseTableInfo> tableInfos) {
Set<MTMV> mtmvs = Sets.newLinkedHashSet();
Set<BaseTableInfo> mvInfos = getMTMVInfos(tableInfos);
for (BaseTableInfo tableInfo : mvInfos) {
try {
- mtmvs.add((MTMV) MTMVUtil.getTable(tableInfo));
+ MTMV mtmv = (MTMV) MTMVUtil.getTable(tableInfo);
+ if (mtmv.canBeCandidate()) {
+ mtmvs.add(mtmv);
+ }
} catch (Exception e) {
// not throw exception to client, just ignore it
LOG.warn("getTable failed: {}", tableInfo.toString(), e);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
index afaad55a34b..58b2a37d504 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
@@ -20,8 +20,6 @@ package org.apache.doris.mtmv;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.Partition;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVRefreshState;
-import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Lists;
@@ -56,8 +54,7 @@ public class MTMVRewriteUtil {
return res;
}
// check mv is normal
- MTMVStatus mtmvStatus = mtmv.getStatus();
- if (mtmvStatus.getState() != MTMVState.NORMAL ||
mtmvStatus.getRefreshState() == MTMVRefreshState.INIT) {
+ if (!mtmv.canBeCandidate()) {
return res;
}
// if relatedPartitions is empty but not null, which means query no
partitions
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java
index b1761b9e973..aa058e628c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java
@@ -82,6 +82,12 @@ public class MTMVStatus {
return this;
}
+ public boolean canBeCandidate() {
+ // MTMVRefreshState.FAIL also can be candidate, because may have some
sync partitions
+ return getState() == MTMVState.NORMAL
+ && getRefreshState() != MTMVRefreshState.INIT;
+ }
+
@Override
public String toString() {
return "MTMVStatus{"
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 17a1abd16b4..104204b89d7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -21,6 +21,7 @@ import org.apache.doris.analysis.StatementBase;
import org.apache.doris.analysis.TableScanParams;
import org.apache.doris.analysis.TableSnapshot;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.View;
@@ -191,6 +192,7 @@ public class StatementContext implements Closeable {
// if query is: select * from t2 join t5
// mtmvRelatedTables is mv1, mv2, mv3, t1, t2, t3, t4, t5
private final Map<List<String>, TableIf> mtmvRelatedTables =
Maps.newHashMap();
+ private final Set<MTMV> candidateMTMVs = Sets.newHashSet();
// insert into target tables
private final Map<List<String>, TableIf> insertTargetTables =
Maps.newHashMap();
// save view's def and sql mode to avoid them change before lock
@@ -321,6 +323,10 @@ public class StatementContext implements Closeable {
return mtmvRelatedTables;
}
+ public Set<MTMV> getCandidateMTMVs() {
+ return candidateMTMVs;
+ }
+
public Map<List<String>, TableIf> getTables() {
return tables;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
index 0e668e94da3..11f8844781d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
@@ -216,11 +216,12 @@ public class CollectRelation implements
AnalysisRuleFactory {
}
if (shouldCollect) {
Set<MTMV> mtmvSet =
Env.getCurrentEnv().getMtmvService().getRelationManager()
- .getAllMTMVs(Lists.newArrayList(new BaseTableInfo(table)));
+ .getCandidateMTMVs(Lists.newArrayList(new
BaseTableInfo(table)));
if (LOG.isDebugEnabled()) {
LOG.debug("table {} related mv set is {}", new
BaseTableInfo(table), mtmvSet);
}
for (MTMV mtmv : mtmvSet) {
+
cascadesContext.getStatementContext().getCandidateMTMVs().add(mtmv);
cascadesContext.getStatementContext().getMtmvRelatedTables().put(mtmv.getFullQualifiers(),
mtmv);
mtmv.readMvLock();
try {
@@ -232,6 +233,7 @@ public class CollectRelation implements AnalysisRuleFactory
{
LOG.debug("mtmv {} related base table include {}",
new BaseTableInfo(mtmv), baseTableInfo);
}
try {
+ // Collect all base tables and lock them before
querying
cascadesContext.getStatementContext().getAndCacheTable(baseTableInfo.toList(),
TableFrom.MTMV, Optional.empty());
} catch (AnalysisException exception) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java
index fbcf4726a10..e86cca263d3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java
@@ -20,16 +20,13 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.TableIf;
-import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVUtil;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.PlannerHook;
import com.google.common.annotations.VisibleForTesting;
-import java.util.List;
import java.util.Set;
-import java.util.stream.Collectors;
/**
* If enable query rewrite with mv in dml, should init consistent
materialization context after analyze
@@ -49,10 +46,9 @@ public class InitConsistentMaterializationContextHook
extends InitMaterializatio
}
protected Set<MTMV> getAvailableMTMVs(Set<TableIf> usedTables,
CascadesContext cascadesContext) {
- List<BaseTableInfo> usedBaseTables =
-
usedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList());
return Env.getCurrentEnv().getMtmvService().getRelationManager()
- .getAvailableMTMVs(usedBaseTables,
cascadesContext.getConnectContext(),
+
.getAvailableMTMVs(cascadesContext.getStatementContext().getCandidateMTMVs(),
+ cascadesContext.getConnectContext(),
true, ((connectContext, mtmv) -> {
return MTMVUtil.mtmvContainsExternalTable(mtmv) &&
(!connectContext.getSessionVariable()
.isEnableDmlMaterializedViewRewriteWhenBaseTableUnawareness());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
index 279ef88481e..f0ef4b11801 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
@@ -26,7 +26,6 @@ import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
-import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVCache;
import org.apache.doris.mtmv.MTMVPlanUtil;
import org.apache.doris.mtmv.MTMVUtil;
@@ -53,7 +52,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import java.util.stream.Collectors;
/**
* If enable query rewrite with mv, should init materialization context after
analyze
@@ -171,10 +169,9 @@ public class InitMaterializationContextHook implements
PlannerHook {
}
protected Set<MTMV> getAvailableMTMVs(Set<TableIf> usedTables,
CascadesContext cascadesContext) {
- List<BaseTableInfo> usedBaseTables =
-
usedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList());
return Env.getCurrentEnv().getMtmvService().getRelationManager()
- .getAvailableMTMVs(usedBaseTables,
cascadesContext.getConnectContext(),
+
.getAvailableMTMVs(cascadesContext.getStatementContext().getCandidateMTMVs(),
+ cascadesContext.getConnectContext(),
false, ((connectContext, mtmv) -> {
return MTMVUtil.mtmvContainsExternalTable(mtmv) &&
(!connectContext.getSessionVariable()
.isEnableMaterializedViewRewriteWhenBaseTableUnawareness());
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java
index 17ec145f583..342f9fd60c8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java
@@ -51,7 +51,7 @@ public class AlterMTMVTest extends TestWithFeService {
MTMVRelationManager relationManager =
Env.getCurrentEnv().getMtmvService().getRelationManager();
Table table =
Env.getCurrentInternalCatalog().getDb("test").get().getTableOrMetaException("stu");
- Set<MTMV> allMTMVs =
relationManager.getAllMTMVs(Lists.newArrayList(new BaseTableInfo(table)));
+ Set<MTMV> allMTMVs =
relationManager.getCandidateMTMVs(Lists.newArrayList(new BaseTableInfo(table)));
boolean hasMvA = false;
boolean hasMvB = false;
for (MTMV mtmv : allMTMVs) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
index 82c7eaac631..e4788c18409 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
@@ -122,6 +122,10 @@ public class MTMVRewriteUtilTest {
MTMVUtil.mtmvContainsExternalTable((MTMV) any);
minTimes = 0;
result = false;
+
+ mtmv.canBeCandidate();
+ minTimes = 0;
+ result = true;
}
};
}
@@ -279,9 +283,9 @@ public class MTMVRewriteUtilTest {
public void testGetMTMVCanRewritePartitionsStateAbnormal() {
new Expectations() {
{
- status.getState();
+ mtmv.canBeCandidate();
minTimes = 0;
- result = MTMVState.SCHEMA_CHANGE;
+ result = false;
}
};
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
@@ -309,9 +313,9 @@ public class MTMVRewriteUtilTest {
public void testGetMTMVCanRewritePartitionsRefreshStateInit() {
new Expectations() {
{
- status.getRefreshState();
+ mtmv.canBeCandidate();
minTimes = 0;
- result = MTMVRefreshState.INIT;
+ result = false;
}
};
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
index 19d1efdbbd8..0d3181d15e8 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
@@ -70,6 +70,12 @@ class StructInfoMapTest extends SqlTestBase {
return true;
}
};
+ new MockUp<MTMV>() {
+ @Mock
+ public boolean canBeCandidate() {
+ return true;
+ }
+ };
connectContext.getSessionVariable().enableMaterializedViewRewrite =
true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite
= true;
@@ -129,6 +135,12 @@ class StructInfoMapTest extends SqlTestBase {
return true;
}
};
+ new MockUp<MTMV>() {
+ @Mock
+ public boolean canBeCandidate() {
+ return true;
+ }
+ };
connectContext.getSessionVariable().enableMaterializedViewRewrite =
true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite
= true;
createMvByNereids("create materialized view mv1 BUILD IMMEDIATE
REFRESH COMPLETE ON MANUAL\n"
@@ -177,6 +189,12 @@ class StructInfoMapTest extends SqlTestBase {
return true;
}
};
+ new MockUp<MTMV>() {
+ @Mock
+ public boolean canBeCandidate() {
+ return true;
+ }
+ };
connectContext.getSessionVariable().enableMaterializedViewRewrite =
true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite
= true;
createMvByNereids("create materialized view mv1 BUILD IMMEDIATE
REFRESH COMPLETE ON MANUAL\n"
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java
index 1403a9fee5e..3e93919ed22 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java
@@ -59,6 +59,12 @@ public class IdStatisticsMapTest extends SqlTestBase {
return true;
}
};
+ new MockUp<MTMV>() {
+ @Mock
+ public boolean canBeCandidate() {
+ return true;
+ }
+ };
connectContext.getSessionVariable().enableMaterializedViewRewrite =
true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite
= true;
createMvByNereids("create materialized view mv100 BUILD IMMEDIATE
REFRESH COMPLETE ON MANUAL\n"
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java
index 4fa0a68e77c..5c9fb3c0392 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java
@@ -55,6 +55,12 @@ public class MvTableIdIsLongTest extends SqlTestBase {
return true;
}
};
+ new MockUp<MTMV>() {
+ @Mock
+ public boolean canBeCandidate() {
+ return true;
+ }
+ };
connectContext.getSessionVariable().enableMaterializedViewRewrite =
true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite
= true;
createMvByNereids("create materialized view mv1 BUILD IMMEDIATE
REFRESH COMPLETE ON MANUAL\n"
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
index 82227632591..bfdbad12b3e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.util;
import org.apache.doris.analysis.ExplainOptions;
+import org.apache.doris.nereids.CTEContext;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.PlanProcess;
@@ -49,6 +50,7 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleFactory;
import org.apache.doris.nereids.rules.RuleSet;
import org.apache.doris.nereids.rules.RuleType;
+import
org.apache.doris.nereids.rules.exploration.mv.InitConsistentMaterializationContextHook;
import
org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
@@ -120,6 +122,9 @@ public class PlanChecker {
}
public PlanChecker analyze() {
+
this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE);
+ this.cascadesContext.newTableCollector().collect();
+ this.cascadesContext.setCteContext(new CTEContext());
this.cascadesContext.newAnalyzer().analyze();
this.cascadesContext.toMemo();
return this;
@@ -127,6 +132,9 @@ public class PlanChecker {
public PlanChecker analyze(Plan plan) {
this.cascadesContext =
MemoTestUtils.createCascadesContext(connectContext, plan);
+
this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE);
+ this.cascadesContext.newTableCollector().collect();
+ this.cascadesContext.setCteContext(new CTEContext());
Set<String> originDisableRules =
connectContext.getSessionVariable().getDisableNereidsRuleNames();
Set<String> disableRuleWithAuth = Sets.newHashSet(originDisableRules);
disableRuleWithAuth.add(RuleType.RELATION_AUTHENTICATION.name());
@@ -140,6 +148,9 @@ public class PlanChecker {
public PlanChecker analyze(String sql) {
this.cascadesContext =
MemoTestUtils.createCascadesContext(connectContext, sql);
+
this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE);
+ this.cascadesContext.newTableCollector().collect();
+ this.cascadesContext.setCteContext(new CTEContext());
this.cascadesContext.newAnalyzer().analyze();
this.cascadesContext.toMemo();
return this;
diff --git a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
index 98477ab92d5..abd0f64450e 100644
--- a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
@@ -54,7 +54,9 @@ suite("test_paimon_mtmv",
"p0,external,mtmv,external_docker,external_docker_dori
"s3.access_key" = "admin",
"s3.secret_key" = "password",
"s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
- "s3.region" = "us-east-1"
+ "s3.region" = "us-east-1",
+ "fs.oss.connection.timeout" = "1000",
+ "fs.oss.connection.establish.timeout" = "1000"
);"""
order_qt_base_table """ select * from
${catalogName}.test_paimon_spark.test_tb_mix_format ; """
diff --git
a/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy
index 9ad559e8d0d..f612c5bb5de 100644
--- a/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy
@@ -56,7 +56,9 @@ suite("test_paimon_olap_rewrite_mtmv",
"p0,external,mtmv,external_docker,externa
"s3.access_key" = "admin",
"s3.secret_key" = "password",
"s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
- "s3.region" = "us-east-1"
+ "s3.region" = "us-east-1",
+ "fs.oss.connection.timeout" = "1000",
+ "fs.oss.connection.establish.timeout" = "1000"
);"""
sql """analyze table ${catalogName}.`test_paimon_spark`.test_tb_mix_format
with sync"""
diff --git a/regression-test/suites/mtmv_p0/test_paimon_rewrite_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_paimon_rewrite_mtmv.groovy
index 024d48da8a7..a1bba2391ed 100644
--- a/regression-test/suites/mtmv_p0/test_paimon_rewrite_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_paimon_rewrite_mtmv.groovy
@@ -39,7 +39,9 @@ suite("test_paimon_rewrite_mtmv",
"p0,external,mtmv,external_docker,external_doc
"s3.access_key" = "admin",
"s3.secret_key" = "password",
"s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
- "s3.region" = "us-east-1"
+ "s3.region" = "us-east-1",
+ "fs.oss.connection.timeout" = "1000",
+ "fs.oss.connection.establish.timeout" = "1000"
);"""
sql """analyze table ${catalogName}.`test_paimon_spark`.test_tb_mix_format
with sync"""
diff --git
a/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy
b/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy
index 54e9b57f7e5..753217f922d 100644
---
a/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy
@@ -106,7 +106,8 @@ suite("dml_insert_and_overwrite") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ // todo: Temporarily turn off, otherwise usable materialized views will
not be collected and will need to be changed back in the future
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
@@ -155,7 +156,7 @@ suite("dml_insert_and_overwrite") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
@@ -203,7 +204,7 @@ suite("dml_insert_and_overwrite") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
@@ -249,7 +250,7 @@ suite("dml_insert_and_overwrite") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
diff --git
a/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy
b/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy
index 350e49057d0..dc23130b990 100644
---
a/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy
@@ -122,7 +122,8 @@ suite("dml_into_outfile", "p0") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ // todo: Temporarily turn off, otherwise usable materialized views will
not be collected and will need to be changed back in the future
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
@@ -185,7 +186,7 @@ suite("dml_into_outfile", "p0") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
