This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 11360b27a24 [fix](catalog) fix potential catalog cache dead lock
#34609 (#34614)
11360b27a24 is described below
commit 11360b27a24a551d62d192111af384a12a4b93e5
Author: Mingyu Chen <[email protected]>
AuthorDate: Sat May 11 18:06:56 2024 +0800
[fix](catalog) fix potential catalog cache dead lock #34609 (#34614)
bp #34609
---
.../java/org/apache/doris/datasource/ExternalCatalog.java | 15 ++++++++++-----
.../org/apache/doris/datasource/ExternalDatabase.java | 15 ++++++++++-----
.../test/java/org/apache/doris/catalog/RefreshDbTest.java | 2 +-
.../org/apache/doris/datasource/RefreshCatalogTest.java | 2 +-
4 files changed, 22 insertions(+), 12 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 435722e06b6..2f8186db722 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -225,9 +225,7 @@ public abstract class ExternalCatalog
initLocalObjects();
if (!initialized) {
if (useMetaCache.get()) {
- if (metaCache != null) {
- metaCache.invalidateAll();
- } else {
+ if (metaCache == null) {
metaCache =
Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache(
name,
OptionalLong.of(86400L),
@@ -344,7 +342,6 @@ public abstract class ExternalCatalog
dbId = dbNameToId.get(dbName);
tmpDbNameToId.put(dbName, dbId);
ExternalDatabase<? extends ExternalTable> db =
idToDb.get(dbId);
- db.setUnInitialized(invalidCacheInInit);
tmpIdToDb.put(dbId, db);
initCatalogLog.addRefreshDb(dbId);
} else {
@@ -379,6 +376,15 @@ public abstract class ExternalCatalog
synchronized (this.propLock) {
this.convertedProperties = null;
}
+ if (useMetaCache.isPresent()) {
+ if (useMetaCache.get() && metaCache != null) {
+ metaCache.invalidateAll();
+ } else if (!useMetaCache.get()) {
+ for (ExternalDatabase<? extends ExternalTable> db :
idToDb.values()) {
+ db.setUnInitialized(invalidCache);
+ }
+ }
+ }
this.invalidCacheInInit = invalidCache;
if (invalidCache) {
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalogCache(id);
@@ -586,7 +592,6 @@ public abstract class ExternalCatalog
// Because replyInitCatalog can only be called when
`use_meta_cache` is false.
// And if `use_meta_cache` is false, getDbForReplay() will not
return null
Preconditions.checkNotNull(db.get());
- db.get().setUnInitialized(invalidCacheInInit);
tmpDbNameToId.put(db.get().getFullName(), db.get().getId());
tmpIdToDb.put(db.get().getId(), db.get());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
index 6ab3421abc3..f34be46d4b2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -118,6 +118,15 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
public void setUnInitialized(boolean invalidCache) {
this.initialized = false;
this.invalidCacheInInit = invalidCache;
+ if (extCatalog.getUseMetaCache().isPresent()) {
+ if (extCatalog.getUseMetaCache().get() && metaCache != null) {
+ metaCache.invalidateAll();
+ } else if (!extCatalog.getUseMetaCache().get()) {
+ for (T table : idToTbl.values()) {
+ table.unsetObjectCreated();
+ }
+ }
+ }
if (invalidCache) {
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDbCache(extCatalog.getId(),
name);
}
@@ -131,9 +140,7 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
extCatalog.makeSureInitialized();
if (!initialized) {
if (extCatalog.getUseMetaCache().get()) {
- if (metaCache != null) {
- metaCache.invalidateAll();
- } else {
+ if (metaCache == null) {
metaCache =
Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache(
name,
OptionalLong.of(86400L),
@@ -175,7 +182,6 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
// So we need add a validation here to avoid table(s) not found,
this is just a temporary solution
// because later we will remove all the logics about
InitCatalogLog/InitDatabaseLog.
if (table.isPresent()) {
- table.get().unsetObjectCreated();
tmpTableNameToId.put(table.get().getName(),
table.get().getId());
tmpIdToTbl.put(table.get().getId(), table.get());
}
@@ -206,7 +212,6 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
tblId = tableNameToId.get(tableName);
tmpTableNameToId.put(tableName, tblId);
T table = idToTbl.get(tblId);
- table.unsetObjectCreated();
tmpIdToTbl.put(tblId, table);
initDatabaseLog.addRefreshTable(tblId);
} else {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java
index 1358e032002..fd920e9632c 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java
@@ -91,7 +91,7 @@ public class RefreshDbTest extends TestWithFeService {
}
long l3 = db1.getLastUpdateTime();
Assertions.assertTrue(l3 == l2);
- Assertions.assertTrue(table.isObjectCreated());
+ Assertions.assertFalse(table.isObjectCreated());
test1.getDbNullable("db1").getTables();
Assertions.assertFalse(table.isObjectCreated());
try {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
index 60345e9c3ad..f64e6523ebf 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
@@ -144,7 +144,7 @@ public class RefreshCatalogTest extends TestWithFeService {
// not triggered init method
long l3 = test2.getLastUpdateTime();
Assertions.assertTrue(l3 == l2);
- Assertions.assertTrue(table.isObjectCreated());
+ Assertions.assertFalse(table.isObjectCreated());
test2.getDbNullable("db1").getTables();
// Assertions.assertFalse(table.isObjectCreated());
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
