This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new e01d051acfd [improvement](external catalog)Optimize the process of
refreshing catalog for 2.1 (#39205) (#39186)
e01d051acfd is described below
commit e01d051acfd3806763ba0948dc5a1aba320452a5
Author: wuwenchi <[email protected]>
AuthorDate: Sat Aug 17 17:02:06 2024 +0800
[improvement](external catalog)Optimize the process of refreshing catalog
for 2.1 (#39205) (#39186)
## Proposed changes
bp: #39205
When the catalog attributes have not changed, refreshing the catalog
only requires processing the cache, without rebuilding the entire
catalog.
---
.../main/java/org/apache/doris/catalog/RefreshManager.java | 2 +-
.../java/org/apache/doris/datasource/ExternalCatalog.java | 9 +++++++++
.../org/apache/doris/datasource/hive/HiveMetadataOps.java | 4 ++--
.../apache/doris/datasource/iceberg/IcebergMetadataOps.java | 4 ++--
.../org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java | 5 +++++
.../java/org/apache/doris/datasource/RefreshCatalogTest.java | 11 -----------
6 files changed, 19 insertions(+), 16 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
index d017ba7829f..52694e5a5bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
@@ -76,7 +76,7 @@ public class RefreshManager {
private void refreshCatalogInternal(CatalogIf catalog, boolean
invalidCache) {
String catalogName = catalog.getName();
if (!catalogName.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
- ((ExternalCatalog) catalog).onRefresh(invalidCache);
+ ((ExternalCatalog) catalog).onRefreshCache(invalidCache);
LOG.info("refresh catalog {} with invalidCache {}", catalogName,
invalidCache);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index b6e94c3d39d..7f8c0c71b51 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -388,6 +388,15 @@ public abstract class ExternalCatalog
synchronized (this.propLock) {
this.convertedProperties = null;
}
+
+ refreshOnlyCatalogCache(invalidCache);
+ }
+
+ public void onRefreshCache(boolean invalidCache) {
+ refreshOnlyCatalogCache(invalidCache);
+ }
+
+ private void refreshOnlyCatalogCache(boolean invalidCache) {
if (useMetaCache.isPresent()) {
if (useMetaCache.get() && metaCache != null) {
metaCache.invalidateAll();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
index dcfc6d1ad33..e855affc31a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
@@ -126,7 +126,7 @@ public class HiveMetadataOps implements ExternalMetadataOps
{
catalogDatabase.setProperties(properties);
catalogDatabase.setComment(properties.getOrDefault("comment", ""));
client.createDatabase(catalogDatabase);
- catalog.onRefresh(true);
+ catalog.onRefreshCache(true);
} catch (Exception e) {
throw new RuntimeException(e.getMessage(), e);
}
@@ -146,7 +146,7 @@ public class HiveMetadataOps implements ExternalMetadataOps
{
}
try {
client.dropDatabase(dbName);
- catalog.onRefresh(true);
+ catalog.onRefreshCache(true);
} catch (Exception e) {
throw new RuntimeException(e.getMessage(), e);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
index a6933f83d76..dd4792715e8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
@@ -107,7 +107,7 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
}
}
nsCatalog.createNamespace(Namespace.of(dbName), properties);
- dorisCatalog.onRefresh(true);
+ dorisCatalog.onRefreshCache(true);
}
@Override
@@ -123,7 +123,7 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
}
SupportsNamespaces nsCatalog = (SupportsNamespaces) catalog;
nsCatalog.dropNamespace(Namespace.of(dbName));
- dorisCatalog.onRefresh(true);
+ dorisCatalog.onRefreshCache(true);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
index 73b6639c7b9..80cc0f554f6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
@@ -121,6 +121,11 @@ public class JdbcExternalCatalog extends ExternalCatalog {
}
}
+ @Override
+ public void onRefreshCache(boolean invalidCache) {
+ onRefresh(invalidCache);
+ }
+
@Override
public void onClose() {
super.onClose();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
index 439385993f9..b51d6b19be5 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
@@ -155,18 +155,7 @@ public class RefreshCatalogTest extends TestWithFeService {
} catch (Exception e) {
// Do nothing
}
- Assertions.assertFalse(((ExternalCatalog) test2).isInitialized());
- table.makeSureInitialized();
Assertions.assertTrue(((ExternalCatalog) test2).isInitialized());
- // table.makeSureInitialized() triggered init method
- long l4 = test2.getLastUpdateTime();
- Assertions.assertTrue(l4 > l3);
- try {
- DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt);
- } catch (Exception e) {
- // Do nothing
- }
- Assertions.assertFalse(((ExternalCatalog) test2).isInitialized());
}
public static class RefreshCatalogProvider implements
TestExternalCatalog.TestCatalogProvider {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
