This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1e7d2fffce6 [fix](catalog) fix deadlock of catalog and database
(#53626)
1e7d2fffce6 is described below
commit 1e7d2fffce6bf0d55dfc891afec220ea223849a9
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Wed Jul 23 00:09:51 2025 -0700
[fix](catalog) fix deadlock of catalog and database (#53626)
### What problem does this PR solve?
Problem Summary:
We should strictly follow the order of `synchronized` of ExternalCatalog
and ExternalDatabase.
First is `ExternalCatalog`, then `ExternalDatabase`.
```
Java stack information for the threads listed above:
===================================================
"STATS_FETCH-3":
at
org.apache.doris.datasource.ExternalCatalog.makeSureInitialized(ExternalCatalog.java:302)
- waiting to lock <0x000000060b1c13f0> (a
org.apache.doris.datasource.jdbc.JdbcExternalCatalog)
at
org.apache.doris.datasource.ExternalDatabase.makeSureInitialized(ExternalDatabase.java:163)
- locked <0x000000060b1c14b0> (a
org.apache.doris.datasource.jdbc.JdbcExternalDatabase)
at
org.apache.doris.datasource.ExternalDatabase.getTableNullable(ExternalDatabase.java:706)
at
org.apache.doris.datasource.ExternalDatabase.getTableNullable(ExternalDatabase.java:72)
at
org.apache.doris.catalog.DatabaseIf.getTableOrException(DatabaseIf.java:154)
at
org.apache.doris.statistics.util.StatisticsUtil.findTable(StatisticsUtil.java:461)
at
org.apache.doris.statistics.ColumnStatisticsCacheLoader.doLoad(ColumnStatisticsCacheLoader.java:41)
at
org.apache.doris.statistics.ColumnStatisticsCacheLoader.doLoad(ColumnStatisticsCacheLoader.java:29)
at
org.apache.doris.statistics.BasicAsyncCacheLoader.lambda$asyncLoad$0(BasicAsyncCacheLoader.java:39)
at
org.apache.doris.statistics.BasicAsyncCacheLoader$$Lambda$2610/0x00007f0b1d4a5c00.get(Unknown
Source)
at
java.util.concurrent.CompletableFuture$AsyncSupply.run([email protected]/CompletableFuture.java:1768)
at
java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/ThreadPoolExecutor.java:1136)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/ThreadPoolExecutor.java:635)
at java.lang.Thread.run([email protected]/Thread.java:840)
"mysql-nio-pool-73":
at
org.apache.doris.datasource.ExternalDatabase.resetToUninitialized(ExternalDatabase.java:135)
- waiting to lock <0x000000060b1c14b0> (a
org.apache.doris.datasource.jdbc.JdbcExternalDatabase)
at
org.apache.doris.datasource.ExternalCatalog.refreshOnlyCatalogCache(ExternalCatalog.java:594)
at
org.apache.doris.datasource.ExternalCatalog.resetToUninitialized(ExternalCatalog.java:579)
- locked <0x000000060b1c13f0> (a
org.apache.doris.datasource.jdbc.JdbcExternalCatalog)
at
org.apache.doris.datasource.jdbc.JdbcExternalCatalog.resetToUninitialized(JdbcExternalCatalog.java:132)
- locked <0x000000060b1c13f0> (a
org.apache.doris.datasource.jdbc.JdbcExternalCatalog)
at
org.apache.doris.catalog.RefreshManager.refreshCatalogInternal(RefreshManager.java:75)
at
org.apache.doris.catalog.RefreshManager.handleRefreshCatalog(RefreshManager.java:58)
at
org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand.run(RefreshCatalogCommand.java:79)
at
org.apache.doris.qe.StmtExecutor.executeByNereids(StmtExecutor.java:707)
at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:545)
at
org.apache.doris.qe.StmtExecutor.queryRetry(StmtExecutor.java:507)
at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:492)
at
org.apache.doris.qe.ConnectProcessor.executeQuery(ConnectProcessor.java:346)
at
org.apache.doris.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:246)
at
org.apache.doris.qe.MysqlConnectProcessor.handleQuery(MysqlConnectProcessor.java:233)
at
org.apache.doris.qe.MysqlConnectProcessor.dispatch(MysqlConnectProcessor.java:261)
at
org.apache.doris.qe.MysqlConnectProcessor.processOnce(MysqlConnectProcessor.java:443)
at
org.apache.doris.mysql.ReadListener.lambda$handleEvent$0(ReadListener.java:52)
at
org.apache.doris.mysql.ReadListener$$Lambda$1018/0x00007f0b1cbc4000.run(Unknown
Source)
at
java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/ThreadPoolExecutor.java:1136)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/ThreadPoolExecutor.java:635)
at java.lang.Thread.run([email protected]/Thread.java:840)
```
---
.../apache/doris/datasource/ExternalDatabase.java | 66 ++++++++++++----------
1 file changed, 35 insertions(+), 31 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
index c440f481034..0f453b1b325 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -154,41 +154,45 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
return initialized;
}
- public final synchronized void makeSureInitialized() {
- if (isInitializing) {
- return;
- }
- isInitializing = true;
- try {
- extCatalog.makeSureInitialized();
- if (!initialized) {
- if (extCatalog.getUseMetaCache().get()) {
- buildMetaCache();
- setLastUpdateTime(System.currentTimeMillis());
- } else {
- if (!Env.getCurrentEnv().isMaster()) {
- // Forward to master and wait the journal to replay.
- int waitTimeOut = ConnectContext.get() == null ? 300 :
ConnectContext.get().getExecTimeoutS();
- MasterCatalogExecutor remoteExecutor = new
MasterCatalogExecutor(waitTimeOut * 1000);
- try {
- remoteExecutor.forward(extCatalog.getId(), id);
- } catch (Exception e) {
- Util.logAndThrowRuntimeException(LOG,
- String.format("failed to forward init
external db %s operation to master", name),
- e);
+ public final void makeSureInitialized() {
+ // Must call this method before any operation on the database to avoid
deadlock of synchronized block
+ extCatalog.makeSureInitialized();
+ synchronized (this) {
+ if (isInitializing) {
+ return;
+ }
+ isInitializing = true;
+ try {
+ if (!initialized) {
+ if (extCatalog.getUseMetaCache().get()) {
+ buildMetaCache();
+ setLastUpdateTime(System.currentTimeMillis());
+ } else {
+ if (!Env.getCurrentEnv().isMaster()) {
+ // Forward to master and wait the journal to
replay.
+ int waitTimeOut = ConnectContext.get() == null ?
300
+ : ConnectContext.get().getExecTimeoutS();
+ MasterCatalogExecutor remoteExecutor = new
MasterCatalogExecutor(waitTimeOut * 1000);
+ try {
+ remoteExecutor.forward(extCatalog.getId(), id);
+ } catch (Exception e) {
+ Util.logAndThrowRuntimeException(LOG,
+ String.format("failed to forward init
external db %s operation to master",
+ name), e);
+ }
+ return;
}
- return;
+ init();
}
- init();
+ initialized = true;
}
- initialized = true;
+ } catch (Exception e) {
+ LOG.warn("failed to init db {}, id {}, isInitializing: {},
initialized: {}",
+ this.name, this.id, isInitializing, initialized, e);
+ initialized = false;
+ } finally {
+ isInitializing = false;
}
- } catch (Exception e) {
- LOG.warn("failed to init db {}, id {}, isInitializing: {},
initialized: {}",
- this.name, this.id, isInitializing, initialized, e);
- initialized = false;
- } finally {
- isInitializing = false;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
