This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 6fe00c7244a [fix](mtmv) fix mtmv deadlock issue (#43376) (#43428)
6fe00c7244a is described below
commit 6fe00c7244ae2d5a52d8e35d1e121d8d932edf27
Author: zhangdong <[email protected]>
AuthorDate: Sat Nov 16 20:49:09 2024 +0800
[fix](mtmv) fix mtmv deadlock issue (#43376) (#43428)
pick: https://github.com/apache/doris/pull/43376
---
.../main/java/org/apache/doris/catalog/MTMV.java | 2 +-
.../java/org/apache/doris/catalog/OlapTable.java | 8 ++++--
.../doris/common/proc/PartitionsProcDir.java | 31 +++++++++++++---------
.../org/apache/doris/mtmv/MTMVPartitionUtil.java | 4 +--
.../org/apache/doris/mtmv/MTMVRelatedTableIf.java | 2 +-
5 files changed, 28 insertions(+), 19 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
index d32853b112b..80d7f48a9ee 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
@@ -351,7 +351,7 @@ public class MTMV extends OlapTable {
*
* @return mvPartitionName ==> mvPartitionKeyDesc
*/
- public Map<String, PartitionKeyDesc> generateMvPartitionDescs() {
+ public Map<String, PartitionKeyDesc> generateMvPartitionDescs() throws
AnalysisException {
Map<String, PartitionItem> mtmvItems = getAndCopyPartitionItems();
Map<String, PartitionKeyDesc> result = Maps.newHashMap();
for (Entry<String, PartitionItem> entry : mtmvItems.entrySet()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 8c7e8b59646..343d3a3ca8d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -106,6 +106,7 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
@@ -3010,8 +3011,11 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf {
}
@Override
- public Map<String, PartitionItem> getAndCopyPartitionItems() {
- readLock();
+ public Map<String, PartitionItem> getAndCopyPartitionItems() throws
AnalysisException {
+ if (!tryReadLock(1, TimeUnit.MINUTES)) {
+ throw new AnalysisException(
+ "get table read lock timeout, database=" +
getQualifiedDbName() + ",table=" + getName());
+ }
try {
Map<String, PartitionItem> res = Maps.newHashMap();
for (Entry<Long, PartitionItem> entry :
getPartitionInfo().getIdToItem(false).entrySet()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
index d0771f3be1c..2d8ed1295b3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
@@ -241,6 +241,16 @@ public class PartitionsProcDir implements ProcDirInterface
{
// get info
List<Pair<List<Comparable>, TRow>> partitionInfos = new
ArrayList<Pair<List<Comparable>, TRow>>();
+ Map<Long, List<String>> partitionsUnSyncTables = null;
+ String mtmvPartitionSyncErrorMsg = null;
+ if (olapTable instanceof MTMV) {
+ try {
+ partitionsUnSyncTables = MTMVPartitionUtil
+ .getPartitionsUnSyncTables((MTMV) olapTable);
+ } catch (AnalysisException e) {
+ mtmvPartitionSyncErrorMsg = e.getMessage();
+ }
+ }
olapTable.readLock();
try {
List<Long> partitionIds;
@@ -258,16 +268,6 @@ public class PartitionsProcDir implements ProcDirInterface
{
}
Joiner joiner = Joiner.on(", ");
- Map<Long, List<String>> partitionsUnSyncTables = null;
- String mtmvPartitionSyncErrorMsg = null;
- if (olapTable instanceof MTMV) {
- try {
- partitionsUnSyncTables = MTMVPartitionUtil
- .getPartitionsUnSyncTables((MTMV) olapTable,
partitionIds);
- } catch (AnalysisException e) {
- mtmvPartitionSyncErrorMsg = e.getMessage();
- }
- }
for (Long partitionId : partitionIds) {
Partition partition = olapTable.getPartition(partitionId);
@@ -363,11 +363,16 @@ public class PartitionsProcDir implements
ProcDirInterface {
if (StringUtils.isEmpty(mtmvPartitionSyncErrorMsg)) {
List<String> partitionUnSyncTables =
partitionsUnSyncTables.getOrDefault(partitionId,
Lists.newArrayList());
- boolean isSync =
CollectionUtils.isEmpty(partitionUnSyncTables);
+ boolean isSync =
partitionsUnSyncTables.containsKey(partitionId) && CollectionUtils.isEmpty(
+ partitionUnSyncTables);
partitionInfo.add(isSync);
trow.addToColumnValue(new TCell().setBoolVal(isSync));
- partitionInfo.add(partitionUnSyncTables.toString());
- trow.addToColumnValue(new
TCell().setStringVal(partitionUnSyncTables.toString()));
+ // The calculation logic of partitionsUnSyncTables is
not protected in the current lock,
+ // so the obtained partition list may not be
consistent with here
+ String unSyncTables =
partitionsUnSyncTables.containsKey(partitionId)
+ ? partitionUnSyncTables.toString() : "not
sure, please try again";
+ partitionInfo.add(unSyncTables);
+ trow.addToColumnValue(new
TCell().setStringVal(unSyncTables));
} else {
partitionInfo.add(false);
trow.addToColumnValue(new TCell().setBoolVal(false));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
index 453014f26d1..14b0f89ac76 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
@@ -231,12 +231,12 @@ public class MTMVPartitionUtil {
* getPartitionsUnSyncTables
*
* @param mtmv
- * @param partitionIds
* @return partitionName ==> UnSyncTableNames
* @throws AnalysisException
*/
- public static Map<Long, List<String>> getPartitionsUnSyncTables(MTMV mtmv,
List<Long> partitionIds)
+ public static Map<Long, List<String>> getPartitionsUnSyncTables(MTMV mtmv)
throws AnalysisException {
+ List<Long> partitionIds = mtmv.getPartitionIds();
Map<Long, List<String>> res = Maps.newHashMap();
MTMVRefreshContext context = MTMVRefreshContext.buildContext(mtmv);
for (Long partitionId : partitionIds) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
index c40485db223..4a8b14603ce 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
@@ -40,7 +40,7 @@ public interface MTMVRelatedTableIf extends TableIf {
*
* @return partitionName->PartitionItem
*/
- Map<String, PartitionItem> getAndCopyPartitionItems();
+ Map<String, PartitionItem> getAndCopyPartitionItems() throws
AnalysisException;
/**
* getPartitionType LIST/RANGE/UNPARTITIONED
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
