This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 0e21dba8171 [opt](catalog) modify some meta cache logic (#38506)
(#39628)
0e21dba8171 is described below
commit 0e21dba817180bcfb491bcf8aff81273c5af45b2
Author: Mingyu Chen <[email protected]>
AuthorDate: Tue Aug 20 21:57:55 2024 +0800
[opt](catalog) modify some meta cache logic (#38506) (#39628)
#38506
---
.../main/java/org/apache/doris/common/Config.java | 20 ++++++++++++--------
.../org/apache/doris/datasource/ExternalCatalog.java | 4 +---
.../apache/doris/datasource/ExternalDatabase.java | 2 +-
.../doris/datasource/hive/HiveMetaStoreCache.java | 2 +-
.../hudi/source/HudiCachedPartitionProcessor.java | 4 ++--
.../datasource/iceberg/IcebergMetadataCache.java | 8 ++++----
.../apache/doris/datasource/metacache/MetaCache.java | 2 +-
7 files changed, 22 insertions(+), 20 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 602e5dffdb2..35fdac59685 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1968,16 +1968,20 @@ public class Config extends ConfigBase {
* Max cache num of hive partition.
* Decrease this value if FE's memory is small
*/
- @ConfField(mutable = false, masterOnly = false)
- public static long max_hive_partition_cache_num = 100000;
+ @ConfField(description = {"Hive Metastore 表级别分区缓存的最大数量。",
+ "Max cache number of partition at table level in Hive Metastore."})
+ public static long max_hive_partition_cache_num = 10000;
- @ConfField(mutable = false, masterOnly = false, description =
{"Hive表名缓存的最大数量。",
- "Max cache number of hive table name list."})
- public static long max_hive_table_cache_num = 1000;
+ @ConfField(description = {"Hudi/Iceberg 表级别缓存的最大数量。",
+ "Max cache number of hudi/iceberg table."})
+ public static long max_external_table_cache_num = 1000;
- @ConfField(mutable = false, masterOnly = false, description = {
- "Hive分区表缓存的最大数量", "Max cache number of hive partition table"
- })
+ @ConfField(description = {"External Catalog 中，Database 和 Table
的实例缓存的最大数量。",
+ "Max cache number of database and table instance in external
catalog."})
+ public static long max_meta_object_cache_num = 1000;
+
+ @ConfField(description = {"Hive分区表缓存的最大数量",
+ "Max cache number of hive partition table"})
public static long max_hive_partition_table_cache_num = 1000;
@ConfField(mutable = false, masterOnly = false, description =
{"获取Hive分区值时候的最大返回数量，-1代表没有限制。",
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 7f8c0c71b51..974f052bb1c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -236,7 +236,7 @@ public abstract class ExternalCatalog
name,
OptionalLong.of(86400L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
- Config.max_hive_table_cache_num,
+ Config.max_meta_object_cache_num,
ignored -> getFilteredDatabaseNames(),
dbName -> Optional.ofNullable(
buildDbForInit(dbName,
Util.genIdByName(name, dbName), logType)),
@@ -667,8 +667,6 @@ public abstract class ExternalCatalog
return new IcebergExternalDatabase(this, dbId, dbName);
case MAX_COMPUTE:
return new MaxComputeExternalDatabase(this, dbId, dbName);
- //case HUDI:
- //return new HudiExternalDatabase(this, dbId, dbName);
case TEST:
return new TestExternalDatabase(this, dbId, dbName);
case PAIMON:
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
index b564a17ce86..dc6f9aaea73 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -145,7 +145,7 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
name,
OptionalLong.of(86400L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
- Config.max_hive_table_cache_num,
+ Config.max_meta_object_cache_num,
ignored -> listTableNames(),
tableName -> Optional.ofNullable(
buildTableForInit(tableName,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index 81541fce49b..312f2382b0d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -179,7 +179,7 @@ public class HiveMetaStoreCache {
CacheFactory fileCacheFactory = new CacheFactory(
OptionalLong.of(fileMetaCacheTtlSecond >=
HMSExternalCatalog.FILE_META_CACHE_TTL_DISABLE_CACHE
- ? fileMetaCacheTtlSecond : 86400L),
+ ? fileMetaCacheTtlSecond : 28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
Config.max_external_file_cache_num,
false,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java
index d9c1c208271..2372b88e0db 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java
@@ -52,9 +52,9 @@ public class HudiCachedPartitionProcessor extends
HudiPartitionProcessor {
this.catalogId = catalogId;
this.executor = executor;
CacheFactory partitionCacheFactory = new CacheFactory(
- OptionalLong.of(86400L),
+ OptionalLong.of(28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
- Config.max_hive_table_cache_num,
+ Config.max_external_table_cache_num,
false,
null);
this.partitionCache = partitionCacheFactory.buildCache(key -> new
TablePartitionValues(), null, executor);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
index dc11a6cacc2..13bd9650978 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
@@ -55,17 +55,17 @@ public class IcebergMetadataCache {
public IcebergMetadataCache(ExecutorService executor) {
CacheFactory snapshotListCacheFactory = new CacheFactory(
- OptionalLong.of(86400L),
+ OptionalLong.of(28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
- Config.max_hive_table_cache_num,
+ Config.max_external_table_cache_num,
false,
null);
this.snapshotListCache = snapshotListCacheFactory.buildCache(key ->
loadSnapshots(key), null, executor);
CacheFactory tableCacheFactory = new CacheFactory(
- OptionalLong.of(86400L),
+ OptionalLong.of(28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
- Config.max_hive_table_cache_num,
+ Config.max_external_table_cache_num,
false,
null);
this.tableCache = tableCacheFactory.buildCache(key -> loadTable(key),
null, executor);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
index c251db3a5c1..e3ad8668fb5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
@@ -57,7 +57,7 @@ public class MetaCache<T> {
CacheFactory namesCacheFactory = new CacheFactory(
expireAfterWriteSec,
refreshAfterWriteSec,
- maxSize,
+ 1, // names cache has one and only one entry
true,
null);
CacheFactory objCacheFactory = new CacheFactory(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
