This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 289100b063a [opt](catalog) modify some meta cache logic (#38506)
289100b063a is described below
commit 289100b063a3f739851477c74970e4b19a56b1db
Author: Mingyu Chen <[email protected]>
AuthorDate: Sat Aug 10 21:22:05 2024 +0800
[opt](catalog) modify some meta cache logic (#38506)
1. Add a new FE config `max_meta_object_cache_num` to control the meta
number of db/table in external catalog
2. Shorten the expire time of some cache to make data more refresh
---
.../main/java/org/apache/doris/common/Config.java | 20 ++++++++++++--------
.../org/apache/doris/datasource/ExternalCatalog.java | 4 +---
.../apache/doris/datasource/ExternalDatabase.java | 2 +-
.../doris/datasource/hive/HiveMetaStoreCache.java | 2 +-
.../hudi/source/HudiCachedPartitionProcessor.java | 4 ++--
.../datasource/iceberg/IcebergMetadataCache.java | 8 ++++----
.../apache/doris/datasource/metacache/MetaCache.java | 2 +-
7 files changed, 22 insertions(+), 20 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index de3c8c2c662..3588a3a0bff 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1979,16 +1979,20 @@ public class Config extends ConfigBase {
* Max cache num of hive partition.
* Decrease this value if FE's memory is small
*/
- @ConfField(mutable = false, masterOnly = false)
- public static long max_hive_partition_cache_num = 100000;
+ @ConfField(description = {"Hive Metastore 表级别分区缓存的最大数量。",
+ "Max cache number of partition at table level in Hive Metastore."})
+ public static long max_hive_partition_cache_num = 10000;
- @ConfField(mutable = false, masterOnly = false, description =
{"Hive表名缓存的最大数量。",
- "Max cache number of hive table name list."})
- public static long max_hive_table_cache_num = 1000;
+ @ConfField(description = {"Hudi/Iceberg 表级别缓存的最大数量。",
+ "Max cache number of hudi/iceberg table."})
+ public static long max_external_table_cache_num = 1000;
- @ConfField(mutable = false, masterOnly = false, description = {
- "Hive分区表缓存的最大数量", "Max cache number of hive partition table"
- })
+ @ConfField(description = {"External Catalog 中，Database 和 Table
的实例缓存的最大数量。",
+ "Max cache number of database and table instance in external
catalog."})
+ public static long max_meta_object_cache_num = 1000;
+
+ @ConfField(description = {"Hive分区表缓存的最大数量",
+ "Max cache number of hive partition table"})
public static long max_hive_partition_table_cache_num = 1000;
@ConfField(mutable = false, masterOnly = false, description =
{"获取Hive分区值时候的最大返回数量，-1代表没有限制。",
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 4f587ab7465..2dfcec1d8ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -238,7 +238,7 @@ public abstract class ExternalCatalog
name,
OptionalLong.of(86400L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
- Config.max_hive_table_cache_num,
+ Config.max_meta_object_cache_num,
ignored -> getFilteredDatabaseNames(),
dbName -> Optional.ofNullable(
buildDbForInit(dbName,
Util.genIdByName(name, dbName), logType)),
@@ -660,8 +660,6 @@ public abstract class ExternalCatalog
return new IcebergExternalDatabase(this, dbId, dbName);
case MAX_COMPUTE:
return new MaxComputeExternalDatabase(this, dbId, dbName);
- //case HUDI:
- //return new HudiExternalDatabase(this, dbId, dbName);
case LAKESOUL:
return new LakeSoulExternalDatabase(this, dbId, dbName);
case TEST:
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
index b564a17ce86..dc6f9aaea73 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -145,7 +145,7 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
name,
OptionalLong.of(86400L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
- Config.max_hive_table_cache_num,
+ Config.max_meta_object_cache_num,
ignored -> listTableNames(),
tableName -> Optional.ofNullable(
buildTableForInit(tableName,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index da88a03f2eb..631362a5b41 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -177,7 +177,7 @@ public class HiveMetaStoreCache {
CacheFactory fileCacheFactory = new CacheFactory(
OptionalLong.of(fileMetaCacheTtlSecond >=
HMSExternalCatalog.FILE_META_CACHE_TTL_DISABLE_CACHE
- ? fileMetaCacheTtlSecond : 86400L),
+ ? fileMetaCacheTtlSecond : 28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
Config.max_external_file_cache_num,
false,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java
index d9c1c208271..2372b88e0db 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java
@@ -52,9 +52,9 @@ public class HudiCachedPartitionProcessor extends
HudiPartitionProcessor {
this.catalogId = catalogId;
this.executor = executor;
CacheFactory partitionCacheFactory = new CacheFactory(
- OptionalLong.of(86400L),
+ OptionalLong.of(28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
- Config.max_hive_table_cache_num,
+ Config.max_external_table_cache_num,
false,
null);
this.partitionCache = partitionCacheFactory.buildCache(key -> new
TablePartitionValues(), null, executor);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
index dc11a6cacc2..13bd9650978 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
@@ -55,17 +55,17 @@ public class IcebergMetadataCache {
public IcebergMetadataCache(ExecutorService executor) {
CacheFactory snapshotListCacheFactory = new CacheFactory(
- OptionalLong.of(86400L),
+ OptionalLong.of(28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
- Config.max_hive_table_cache_num,
+ Config.max_external_table_cache_num,
false,
null);
this.snapshotListCache = snapshotListCacheFactory.buildCache(key ->
loadSnapshots(key), null, executor);
CacheFactory tableCacheFactory = new CacheFactory(
- OptionalLong.of(86400L),
+ OptionalLong.of(28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
- Config.max_hive_table_cache_num,
+ Config.max_external_table_cache_num,
false,
null);
this.tableCache = tableCacheFactory.buildCache(key -> loadTable(key),
null, executor);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
index c251db3a5c1..e3ad8668fb5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
@@ -57,7 +57,7 @@ public class MetaCache<T> {
CacheFactory namesCacheFactory = new CacheFactory(
expireAfterWriteSec,
refreshAfterWriteSec,
- maxSize,
+ 1, // names cache has one and only one entry
true,
null);
CacheFactory objCacheFactory = new CacheFactory(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
