This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new da7b2cf5783 [refactor](catalog) set "use_meta_cache" default to true
(#38244)(#38352)(#38619) (#38355)
da7b2cf5783 is described below
commit da7b2cf5783f2c70881e396e2d29eb17834e6e76
Author: Mingyu Chen <[email protected]>
AuthorDate: Fri Aug 2 14:13:38 2024 +0800
[refactor](catalog) set "use_meta_cache" default to true
(#38244)(#38352)(#38619) (#38355)
bp #38244 #38352 #38619
---------
Co-authored-by: Yulei-Yang <[email protected]>
---
.../main/java/org/apache/doris/common/util/Util.java | 8 ++++----
.../java/org/apache/doris/datasource/CatalogMgr.java | 6 +++---
.../org/apache/doris/datasource/ExternalCatalog.java | 19 +++++++++++++++----
.../org/apache/doris/datasource/ExternalDatabase.java | 11 +++++++++--
.../datasource/hive/event/AlterDatabaseEvent.java | 2 +-
.../datasource/hive/event/CreateDatabaseEvent.java | 2 +-
.../apache/doris/datasource/metacache/MetaCache.java | 7 +++----
.../java/org/apache/doris/catalog/RefreshDbTest.java | 3 +++
.../apache/doris/datasource/RefreshCatalogTest.java | 5 ++++-
.../iceberg/test_iceberg_table_stats.groovy | 4 ++++
10 files changed, 47 insertions(+), 20 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
index 79b54508771..050ab7774e0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
@@ -662,9 +662,9 @@ public class Util {
}
}
- // Only used for external table's id generation
- // And the table's id must >=0, see DescriptorTable.toThrift()
- public static long genTableIdByName(String tblName) {
- return Math.abs(sha256long(tblName));
+ // Only used for external db/table's id generation
+ // And the db/table's id must >=0, see DescriptorTable.toThrift()
+ public static long genIdByName(String... names) {
+ return Math.abs(sha256long(String.join(".", names)));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index b2c60f2cc14..a8aa34cf6c8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -665,7 +665,7 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
long tblId;
HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog;
if (hmsCatalog.getUseMetaCache().get()) {
- tblId = Util.genTableIdByName(tableName);
+ tblId = Util.genIdByName(catalogName, dbName, tableName);
} else {
tblId =
Env.getCurrentEnv().getExternalMetaIdMgr().getTblId(catalog.getId(), dbName,
tableName);
}
@@ -703,7 +703,7 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
((HMSExternalCatalog) catalog).unregisterDatabase(dbName);
}
- public void registerExternalDatabase(String dbName, String catalogName,
boolean ignoreIfExists)
+ public void registerExternalDatabaseFromEvent(String dbName, String
catalogName, boolean ignoreIfExists)
throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
@@ -723,7 +723,7 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog;
long dbId;
if (hmsCatalog.getUseMetaCache().get()) {
- dbId = Util.genTableIdByName(dbName);
+ dbId = Util.genIdByName(catalogName, dbName);
} else {
dbId =
Env.getCurrentEnv().getExternalMetaIdMgr().getDbId(catalog.getId(), dbName);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 35434a43cb3..b6e94c3d39d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -99,8 +99,7 @@ public abstract class ExternalCatalog
public static final String DORIS_VERSION = "doris.version";
public static final String DORIS_VERSION_VALUE =
Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH;
public static final String USE_META_CACHE = "use_meta_cache";
- // Set default value to false to be compatible with older version meta
data.
- public static final boolean DEFAULT_USE_META_CACHE = false;
+ public static final boolean DEFAULT_USE_META_CACHE = true;
// Unique id of this catalog, will be assigned after catalog is loaded.
@SerializedName(value = "id")
@@ -240,7 +239,7 @@ public abstract class ExternalCatalog
Config.max_hive_table_cache_num,
ignored -> getFilteredDatabaseNames(),
dbName -> Optional.ofNullable(
- buildDbForInit(dbName,
Util.genTableIdByName(dbName), logType)),
+ buildDbForInit(dbName,
Util.genIdByName(name, dbName), logType)),
(key, value, cause) -> value.ifPresent(v ->
v.setUnInitialized(invalidCacheInInit)));
}
setLastUpdateTime(System.currentTimeMillis());
@@ -289,6 +288,12 @@ public abstract class ExternalCatalog
throw new DdlException("Invalid properties: " +
CatalogMgr.METADATA_REFRESH_INTERVAL_SEC);
}
}
+
+ if (properties.getOrDefault(ExternalCatalog.USE_META_CACHE,
"true").equals("false")) {
+ LOG.warn("force to set use_meta_cache to true for catalog: {} when
creating", name);
+ getCatalogProperty().addProperty(ExternalCatalog.USE_META_CACHE,
"true");
+ useMetaCache = Optional.of(true);
+ }
}
/**
@@ -497,7 +502,9 @@ public abstract class ExternalCatalog
}
if (useMetaCache.get()) {
- return metaCache.getMetaObj(realDbName).orElse(null);
+ // must use full qualified name to generate id.
+ // otherwise, if 2 catalogs have the same db name, the id will be
the same.
+ return metaCache.getMetaObj(realDbName,
Util.genIdByName(getQualifiedName(realDbName))).orElse(null);
} else {
if (dbNameToId.containsKey(realDbName)) {
return idToDb.get(dbNameToId.get(realDbName));
@@ -854,4 +861,8 @@ public abstract class ExternalCatalog
throw e;
}
}
+
+ public String getQualifiedName(String dbName) {
+ return String.join(".", name, dbName);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
index f34be46d4b2..2919633858f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -148,7 +148,8 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
Config.max_hive_table_cache_num,
ignored -> listTableNames(),
tableName -> Optional.ofNullable(
- buildTableForInit(tableName,
Util.genTableIdByName(tableName), extCatalog)),
+ buildTableForInit(tableName,
+
Util.genIdByName(extCatalog.getName(), name, tableName), extCatalog)),
(key, value, cause) ->
value.ifPresent(ExternalTable::unsetObjectCreated));
}
setLastUpdateTime(System.currentTimeMillis());
@@ -375,7 +376,9 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
public T getTableNullable(String tableName) {
makeSureInitialized();
if (extCatalog.getUseMetaCache().get()) {
- return metaCache.getMetaObj(tableName).orElse(null);
+ // must use full qualified name to generate id.
+ // otherwise, if 2 databases have the same table name, the id will
be the same.
+ return metaCache.getMetaObj(tableName,
Util.genIdByName(getQualifiedName(tableName))).orElse(null);
} else {
if (!tableNameToId.containsKey(tableName)) {
return null;
@@ -493,4 +496,8 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
setLastUpdateTime(System.currentTimeMillis());
return true;
}
+
+ public String getQualifiedName(String tblName) {
+ return String.join(".", extCatalog.getName(), name, tblName);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java
index c4529d5d487..ef828025e7f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java
@@ -88,7 +88,7 @@ public class AlterDatabaseEvent extends MetastoreEvent {
return;
}
Env.getCurrentEnv().getCatalogMgr().unregisterExternalDatabase(dbBefore.getName(),
catalogName, true);
-
Env.getCurrentEnv().getCatalogMgr().registerExternalDatabase(dbAfter.getName(),
catalogName, true);
+
Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(dbAfter.getName(),
catalogName, true);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
index 8829d2ae988..2dd4c5671bc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
@@ -55,7 +55,7 @@ public class CreateDatabaseEvent extends MetastoreEvent {
protected void process() throws MetastoreNotificationException {
try {
infoLog("catalogName:[{}],dbName:[{}]", catalogName, dbName);
-
Env.getCurrentEnv().getCatalogMgr().registerExternalDatabase(dbName,
catalogName, true);
+
Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(dbName,
catalogName, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"), e);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
index da8f068dfd4..c251db3a5c1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
@@ -18,7 +18,6 @@
package org.apache.doris.datasource.metacache;
import org.apache.doris.common.CacheFactory;
-import org.apache.doris.common.util.Util;
import com.github.benmanes.caffeine.cache.CacheLoader;
import com.github.benmanes.caffeine.cache.LoadingCache;
@@ -75,12 +74,12 @@ public class MetaCache<T> {
return namesCache.get("");
}
- public Optional<T> getMetaObj(String name) {
+ public Optional<T> getMetaObj(String name, long id) {
Optional<T> val = metaObjCache.getIfPresent(name);
if (val == null) {
synchronized (metaObjCache) {
val = metaObjCache.get(name);
- idToName.put(Util.genTableIdByName(name), name);
+ idToName.put(id, name);
}
}
return val;
@@ -88,7 +87,7 @@ public class MetaCache<T> {
public Optional<T> getMetaObjById(long id) {
String name = idToName.get(id);
- return name == null ? Optional.empty() : getMetaObj(name);
+ return name == null ? Optional.empty() : getMetaObj(name, id);
}
public void updateCache(String objName, T obj) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java
index fd920e9632c..fd7a6b1a24b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java
@@ -91,6 +91,9 @@ public class RefreshDbTest extends TestWithFeService {
}
long l3 = db1.getLastUpdateTime();
Assertions.assertTrue(l3 == l2);
+ // when use_meta_cache is true, the table will be recreated after
refresh.
+ // so we need to get table again
+ table = db1.getTable("tbl11").get();
Assertions.assertFalse(table.isObjectCreated());
test1.getDbNullable("db1").getTables();
Assertions.assertFalse(table.isObjectCreated());
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
index f64e6523ebf..439385993f9 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
@@ -144,9 +144,12 @@ public class RefreshCatalogTest extends TestWithFeService {
// not triggered init method
long l3 = test2.getLastUpdateTime();
Assertions.assertTrue(l3 == l2);
+ // when use_meta_cache is true, the table will be recreated after
refresh.
+ // so we need to get table again
+ table = (TestExternalTable)
test2.getDbNullable("db1").getTable("tbl11").get();
Assertions.assertFalse(table.isObjectCreated());
test2.getDbNullable("db1").getTables();
- // Assertions.assertFalse(table.isObjectCreated());
+ Assertions.assertFalse(table.isObjectCreated());
try {
DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt);
} catch (Exception e) {
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy
index b8eacf6d9e2..15c23fa74d8 100644
---
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy
@@ -48,6 +48,10 @@ suite("test_iceberg_table_stats",
"p0,external,doris,external_docker,external_do
Thread.sleep(2000)
retry++
}
+ if (cnt != act) {
+ def result2 = sql """select * from ${table_name} order by
1 limit 10;"""
+ print result2
+ }
assertEquals(act, cnt)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
