This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 55455328b93 [refactor](catalog) set "use_meta_cache" default to true
(#38244)
55455328b93 is described below
commit 55455328b93f46462a59065f1f1cc69e1ffde194
Author: Mingyu Chen <[email protected]>
AuthorDate: Wed Jul 24 17:06:43 2024 +0800
[refactor](catalog) set "use_meta_cache" default to true (#38244)
This is PR #33610 introduce a new feature of `use_meta_cache=true`.
Now I will set this property default to true.
For all newly created catalog, is this property is not specified, it
will be true.
For all previously created catalog, this property is still false.
---
.../main/java/org/apache/doris/common/util/Util.java | 8 ++++----
.../java/org/apache/doris/datasource/CatalogMgr.java | 6 +++---
.../org/apache/doris/datasource/ExternalCatalog.java | 18 ++++++++++++++----
.../org/apache/doris/datasource/ExternalDatabase.java | 11 +++++++++--
.../datasource/hive/event/AlterDatabaseEvent.java | 2 +-
.../datasource/hive/event/CreateDatabaseEvent.java | 2 +-
.../apache/doris/datasource/metacache/MetaCache.java | 7 +++----
.../apache/doris/datasource/RefreshCatalogTest.java | 2 +-
8 files changed, 36 insertions(+), 20 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
index e73df139e01..c577f778860 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
@@ -677,9 +677,9 @@ public class Util {
}
}
- // Only used for external table's id generation
- // And the table's id must >=0, see DescriptorTable.toThrift()
- public static long genTableIdByName(String tblName) {
- return Math.abs(sha256long(tblName));
+ // Only used for external db/table's id generation
+ // And the db/table's id must >=0, see DescriptorTable.toThrift()
+ public static long genIdByName(String... names) {
+ return Math.abs(sha256long(String.join(".", names)));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index 2aec7655fe5..8ed602d0eb8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -665,7 +665,7 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
long tblId;
HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog;
if (hmsCatalog.getUseMetaCache().get()) {
- tblId = Util.genTableIdByName(tableName);
+ tblId = Util.genIdByName(catalogName, dbName, tableName);
} else {
tblId =
Env.getCurrentEnv().getExternalMetaIdMgr().getTblId(catalog.getId(), dbName,
tableName);
}
@@ -703,7 +703,7 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
((HMSExternalCatalog) catalog).unregisterDatabase(dbName);
}
- public void registerExternalDatabase(String dbName, String catalogName,
boolean ignoreIfExists)
+ public void registerExternalDatabaseFromEvent(String dbName, String
catalogName, boolean ignoreIfExists)
throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
@@ -723,7 +723,7 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog;
long dbId;
if (hmsCatalog.getUseMetaCache().get()) {
- dbId = Util.genTableIdByName(dbName);
+ dbId = Util.genIdByName(catalogName, dbName);
} else {
dbId =
Env.getCurrentEnv().getExternalMetaIdMgr().getDbId(catalog.getId(), dbName);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 4ba5a2ebd96..a425c782fe3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -101,8 +101,7 @@ public abstract class ExternalCatalog
public static final String DORIS_VERSION = "doris.version";
public static final String DORIS_VERSION_VALUE =
Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH;
public static final String USE_META_CACHE = "use_meta_cache";
- // Set default value to false to be compatible with older version meta
data.
- public static final boolean DEFAULT_USE_META_CACHE = false;
+ public static final boolean DEFAULT_USE_META_CACHE = true;
// Unique id of this catalog, will be assigned after catalog is loaded.
@SerializedName(value = "id")
@@ -242,7 +241,7 @@ public abstract class ExternalCatalog
Config.max_hive_table_cache_num,
ignored -> getFilteredDatabaseNames(),
dbName -> Optional.ofNullable(
- buildDbForInit(dbName,
Util.genTableIdByName(dbName), logType)),
+ buildDbForInit(dbName,
Util.genIdByName(name, dbName), logType)),
(key, value, cause) -> value.ifPresent(v ->
v.setUnInitialized(invalidCacheInInit)));
}
setLastUpdateTime(System.currentTimeMillis());
@@ -291,6 +290,11 @@ public abstract class ExternalCatalog
throw new DdlException("Invalid properties: " +
CatalogMgr.METADATA_REFRESH_INTERVAL_SEC);
}
}
+
+ if (properties.getOrDefault(ExternalCatalog.USE_META_CACHE,
"true").equals("false")) {
+ LOG.warn("force to set use_meta_cache to true for catalog: {} when
creating", name);
+ getCatalogProperty().addProperty(ExternalCatalog.USE_META_CACHE,
"true");
+ }
}
/**
@@ -499,7 +503,9 @@ public abstract class ExternalCatalog
}
if (useMetaCache.get()) {
- return metaCache.getMetaObj(realDbName).orElse(null);
+ // must use full qualified name to generate id.
+ // otherwise, if 2 catalogs have the same db name, the id will be
the same.
+ return metaCache.getMetaObj(realDbName,
Util.genIdByName(getQualifiedName(realDbName))).orElse(null);
} else {
if (dbNameToId.containsKey(realDbName)) {
return idToDb.get(dbNameToId.get(realDbName));
@@ -860,4 +866,8 @@ public abstract class ExternalCatalog
throw e;
}
}
+
+ public String getQualifiedName(String dbName) {
+ return String.join(".", name, dbName);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
index f34be46d4b2..2919633858f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -148,7 +148,8 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
Config.max_hive_table_cache_num,
ignored -> listTableNames(),
tableName -> Optional.ofNullable(
- buildTableForInit(tableName,
Util.genTableIdByName(tableName), extCatalog)),
+ buildTableForInit(tableName,
+
Util.genIdByName(extCatalog.getName(), name, tableName), extCatalog)),
(key, value, cause) ->
value.ifPresent(ExternalTable::unsetObjectCreated));
}
setLastUpdateTime(System.currentTimeMillis());
@@ -375,7 +376,9 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
public T getTableNullable(String tableName) {
makeSureInitialized();
if (extCatalog.getUseMetaCache().get()) {
- return metaCache.getMetaObj(tableName).orElse(null);
+ // must use full qualified name to generate id.
+ // otherwise, if 2 databases have the same table name, the id will
be the same.
+ return metaCache.getMetaObj(tableName,
Util.genIdByName(getQualifiedName(tableName))).orElse(null);
} else {
if (!tableNameToId.containsKey(tableName)) {
return null;
@@ -493,4 +496,8 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
setLastUpdateTime(System.currentTimeMillis());
return true;
}
+
+ public String getQualifiedName(String tblName) {
+ return String.join(".", extCatalog.getName(), name, tblName);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java
index c4529d5d487..ef828025e7f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java
@@ -88,7 +88,7 @@ public class AlterDatabaseEvent extends MetastoreEvent {
return;
}
Env.getCurrentEnv().getCatalogMgr().unregisterExternalDatabase(dbBefore.getName(),
catalogName, true);
-
Env.getCurrentEnv().getCatalogMgr().registerExternalDatabase(dbAfter.getName(),
catalogName, true);
+
Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(dbAfter.getName(),
catalogName, true);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
index 8829d2ae988..2dd4c5671bc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
@@ -55,7 +55,7 @@ public class CreateDatabaseEvent extends MetastoreEvent {
protected void process() throws MetastoreNotificationException {
try {
infoLog("catalogName:[{}],dbName:[{}]", catalogName, dbName);
-
Env.getCurrentEnv().getCatalogMgr().registerExternalDatabase(dbName,
catalogName, true);
+
Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(dbName,
catalogName, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"), e);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
index da8f068dfd4..c251db3a5c1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
@@ -18,7 +18,6 @@
package org.apache.doris.datasource.metacache;
import org.apache.doris.common.CacheFactory;
-import org.apache.doris.common.util.Util;
import com.github.benmanes.caffeine.cache.CacheLoader;
import com.github.benmanes.caffeine.cache.LoadingCache;
@@ -75,12 +74,12 @@ public class MetaCache<T> {
return namesCache.get("");
}
- public Optional<T> getMetaObj(String name) {
+ public Optional<T> getMetaObj(String name, long id) {
Optional<T> val = metaObjCache.getIfPresent(name);
if (val == null) {
synchronized (metaObjCache) {
val = metaObjCache.get(name);
- idToName.put(Util.genTableIdByName(name), name);
+ idToName.put(id, name);
}
}
return val;
@@ -88,7 +87,7 @@ public class MetaCache<T> {
public Optional<T> getMetaObjById(long id) {
String name = idToName.get(id);
- return name == null ? Optional.empty() : getMetaObj(name);
+ return name == null ? Optional.empty() : getMetaObj(name, id);
}
public void updateCache(String objName, T obj) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
index 380b1d71354..87906261b24 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
@@ -145,7 +145,7 @@ public class RefreshCatalogTest extends TestWithFeService {
// not triggered init method
long l3 = test2.getLastUpdateTime();
Assertions.assertTrue(l3 == l2);
- Assertions.assertFalse(table.isObjectCreated());
+ // Assertions.assertFalse(table.isObjectCreated());
test2.getDbNullable("db1").getTables();
// Assertions.assertFalse(table.isObjectCreated());
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
