This is an automated email from the ASF dual-hosted git repository.
suxiaogang223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f2ba19ae8f0 [improvement](fe) Improve external catalog meta cache
observability (#63809)
f2ba19ae8f0 is described below
commit f2ba19ae8f04d9905653ed084d6b35d14339c8b7
Author: Socrates <[email protected]>
AuthorDate: Wed Jun 3 10:43:53 2026 +0800
[improvement](fe) Improve external catalog meta cache observability (#63809)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
External catalog meta cache statistics exposed cumulative eviction
count, but did not provide a direct replacement frequency metric for
judging whether cache capacity is too small. This PR adds
`EVICTION_RATE` to `information_schema.catalog_meta_cache_statistics`,
calculated as `eviction_count / request_count` and returned as `0` when
there are no requests.
Hive partition metadata cache defaults were also too small for common
external catalog workloads, causing frequent evictions without explicit
tuning. This PR increases the default Hive single-partition cache
capacity from 10,000 to 100,000 and the Hive partitioned-table values
cache capacity from 1,000 to 10,000. While checking similar cache
entries, MaxCompute `partition_values` was found to cache table-level
partition value structures but use the Hive single-partition capacity;
it now follows the table-level partition values capacity.
### Release note
Add `EVICTION_RATE` to
`information_schema.catalog_meta_cache_statistics`, increase default
Hive partition meta cache capacities, and make MaxCompute
`partition_values` use the table-level partition values capacity.
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [x] Unit Test
- `./run-fe-ut.sh --run
org.apache.doris.datasource.metacache.MetaCacheEntryTest`
- `./run-fe-ut.sh --run
org.apache.doris.datasource.hive.HiveMetaStoreCacheTest,org.apache.doris.datasource.maxcompute.MaxComputeExternalMetaCacheTest`
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [x] Yes. `catalog_meta_cache_statistics` includes `EVICTION_RATE`;
default Hive partition meta cache capacities are larger; MaxCompute
`partition_values` uses the table-level partition values capacity.
- Does this need documentation?
- [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../schema_catalog_meta_cache_stats_scanner.cpp | 1 +
.../main/java/org/apache/doris/common/Config.java | 4 +-
.../src/main/java/org/apache/doris/DorisFE.java | 4 +-
.../java/org/apache/doris/catalog/SchemaTable.java | 1 +
.../maxcompute/MaxComputeExternalMetaCache.java | 2 +-
.../datasource/metacache/MetaCacheEntryStats.java | 7 ++++
.../doris/tablefunction/MetadataGenerator.java | 1 +
.../datasource/hive/HiveMetaStoreCacheTest.java | 43 ++++++++++++++++++++++
.../MaxComputeExternalMetaCacheTest.java | 28 ++++++++++++++
.../datasource/metacache/MetaCacheEntryTest.java | 23 ++++++++++++
10 files changed, 109 insertions(+), 5 deletions(-)
diff --git
a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp
b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp
index d00c93a4c4e..e33590fc69e 100644
--- a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp
+++ b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp
@@ -48,6 +48,7 @@ std::vector<SchemaScanner::ColumnDesc>
SchemaCatalogMetaCacheStatsScanner::_s_tb
{"TOTAL_LOAD_TIME_MS", TYPE_BIGINT, sizeof(int64_t), true},
{"AVG_LOAD_PENALTY_MS", TYPE_DOUBLE, sizeof(double), true},
{"EVICTION_COUNT", TYPE_BIGINT, sizeof(int64_t), true},
+ {"EVICTION_RATE", TYPE_DOUBLE, sizeof(double), true},
{"INVALIDATE_COUNT", TYPE_BIGINT, sizeof(int64_t), true},
{"LAST_LOAD_SUCCESS_TIME", TYPE_STRING, sizeof(StringRef), true},
{"LAST_LOAD_FAILURE_TIME", TYPE_STRING, sizeof(StringRef), true},
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 121503e56c4..d4f3373b9ac 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2160,7 +2160,7 @@ public class Config extends ConfigBase {
* Decrease this value if FE's memory is small
*/
@ConfField(description = {"Maximum cache number of partitions at table
level in Hive Metastore."})
- public static long max_hive_partition_cache_num = 10000;
+ public static long max_hive_partition_cache_num = 100000;
@ConfField(description = {"Maximum cache number of Hudi/Iceberg tables."})
public static long max_external_table_cache_num = 1000;
@@ -2169,7 +2169,7 @@ public class Config extends ConfigBase {
public static long max_meta_object_cache_num = 1000;
@ConfField(description = {"Maximum cache number of Hive partitioned
tables."})
- public static long max_hive_partition_table_cache_num = 1000;
+ public static long max_hive_partition_table_cache_num = 10000;
@ConfField(mutable = false, masterOnly = false, description = {
"Max number of hive partition values to return while list
partitions, -1 means no limitation."})
diff --git a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java
b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java
index a3f5df40841..c5d6e1083ca 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java
@@ -639,8 +639,8 @@ public class DorisFE {
return;
}
- Config.max_hive_partition_cache_num = Util.getRandomLong(0, 10, 10000);
- Config.max_hive_partition_table_cache_num = Util.getRandomLong(0, 10,
10000);
+ Config.max_hive_partition_cache_num = Util.getRandomLong(0, 10, 10000,
100000);
+ Config.max_hive_partition_table_cache_num = Util.getRandomLong(0, 10,
1000, 10000);
Config.external_cache_expire_time_seconds_after_access =
Util.getRandomLong(0, 1, 10, 86400);
Config.external_cache_refresh_time_minutes = Util.getRandomLong(1, 10);
Config.max_external_cache_loader_thread_pool_size =
Util.getRandomInt(1, 10, 64);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
index 77d71e1d459..fdcb838ef6f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
@@ -644,6 +644,7 @@ public class SchemaTable extends Table {
.column("TOTAL_LOAD_TIME_MS",
ScalarType.createType(PrimitiveType.BIGINT))
.column("AVG_LOAD_PENALTY_MS",
ScalarType.createType(PrimitiveType.DOUBLE))
.column("EVICTION_COUNT",
ScalarType.createType(PrimitiveType.BIGINT))
+ .column("EVICTION_RATE",
ScalarType.createType(PrimitiveType.DOUBLE))
.column("INVALIDATE_COUNT",
ScalarType.createType(PrimitiveType.BIGINT))
.column("LAST_LOAD_SUCCESS_TIME",
ScalarType.createStringType())
.column("LAST_LOAD_FAILURE_TIME",
ScalarType.createStringType())
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java
index 46e58f1e380..05bf7e51e30 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java
@@ -60,7 +60,7 @@ public class MaxComputeExternalMetaCache extends
AbstractExternalMetaCache {
CacheSpec.of(
true,
Config.external_cache_refresh_time_minutes * 60L,
- Config.max_hive_partition_cache_num),
+ Config.max_hive_partition_table_cache_num),
MetaCacheEntryInvalidation.forNameMapping(nameMapping ->
nameMapping)));
schemaEntry = registerEntry(MetaCacheEntryDef.of(
ENTRY_SCHEMA,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java
index 495fd011083..0c8b875e730 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java
@@ -162,6 +162,13 @@ public final class MetaCacheEntryStats {
return evictionCount;
}
+ public double getEvictionRate() {
+ if (requestCount == 0) {
+ return 0D;
+ }
+ return (double) evictionCount / requestCount;
+ }
+
public long getInvalidateCount() {
return invalidateCount;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
index 85911b3f3ae..f4d9d535f84 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
@@ -1830,6 +1830,7 @@ public class MetadataGenerator {
trow.addToColumnValue(new TCell().setDoubleVal(
entryStats.getAverageLoadPenaltyNanos() /
TimeUnit.MILLISECONDS.toNanos(1)));
trow.addToColumnValue(new
TCell().setLongVal(entryStats.getEvictionCount())); // EVICTION_COUNT
+ trow.addToColumnValue(new
TCell().setDoubleVal(entryStats.getEvictionRate())); // EVICTION_RATE
trow.addToColumnValue(new
TCell().setLongVal(entryStats.getInvalidateCount())); // INVALIDATE_COUNT
trow.addToColumnValue(new TCell().setStringVal(
formatMetaCacheTime(entryStats.getLastLoadSuccessTimeMs(), timeZone)));
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java
index 04b5bdde1dc..eb4b28e9a88 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java
@@ -17,16 +17,20 @@
package org.apache.doris.datasource.hive;
+import org.apache.doris.common.Config;
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.NameMapping;
import org.apache.doris.datasource.metacache.MetaCacheEntry;
+import org.apache.doris.datasource.metacache.MetaCacheEntryStats;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicLong;
@@ -80,6 +84,45 @@ public class HiveMetaStoreCacheTest {
Assertions.assertEquals(0, entrySize(partitionValuesCache));
}
+ @Test
+ public void testDefaultSpecsFollowConfig() {
+ ThreadPoolExecutor executor =
ThreadPoolManager.newDaemonFixedThreadPool(
+ 1, 1, "refresh", 1, false);
+ ThreadPoolExecutor listExecutor =
ThreadPoolManager.newDaemonFixedThreadPool(
+ 1, 1, "file", 1, false);
+ long originalExpireAfterAccess =
Config.external_cache_expire_time_seconds_after_access;
+ long originalPartitionCapacity = Config.max_hive_partition_cache_num;
+ long originalPartitionTableCapacity =
Config.max_hive_partition_table_cache_num;
+ long originalFileCapacity = Config.max_external_file_cache_num;
+ try {
+ Config.external_cache_expire_time_seconds_after_access = 321L;
+ Config.max_hive_partition_cache_num = 100L;
+ Config.max_hive_partition_table_cache_num = 20L;
+ Config.max_external_file_cache_num = 30L;
+
+ HiveExternalMetaCache hiveMetaStoreCache = new
HiveExternalMetaCache(executor, listExecutor);
+ hiveMetaStoreCache.initCatalog(0, Collections.emptyMap());
+
+ Map<String, MetaCacheEntryStats> stats =
hiveMetaStoreCache.stats(0);
+ MetaCacheEntryStats partitionValuesStats =
stats.get(HiveExternalMetaCache.ENTRY_PARTITION_VALUES);
+ MetaCacheEntryStats partitionStats =
stats.get(HiveExternalMetaCache.ENTRY_PARTITION);
+ MetaCacheEntryStats fileStats =
stats.get(HiveExternalMetaCache.ENTRY_FILE);
+ Assertions.assertEquals(321L, partitionValuesStats.getTtlSecond());
+ Assertions.assertEquals(20L, partitionValuesStats.getCapacity());
+ Assertions.assertEquals(321L, partitionStats.getTtlSecond());
+ Assertions.assertEquals(100L, partitionStats.getCapacity());
+ Assertions.assertEquals(321L, fileStats.getTtlSecond());
+ Assertions.assertEquals(30L, fileStats.getCapacity());
+ } finally {
+ Config.external_cache_expire_time_seconds_after_access =
originalExpireAfterAccess;
+ Config.max_hive_partition_cache_num = originalPartitionCapacity;
+ Config.max_hive_partition_table_cache_num =
originalPartitionTableCapacity;
+ Config.max_external_file_cache_num = originalFileCapacity;
+ executor.shutdownNow();
+ listExecutor.shutdownNow();
+ }
+ }
+
private void putCache(
MetaCacheEntry<HiveExternalMetaCache.FileCacheKey,
HiveExternalMetaCache.FileCacheValue> fileCache,
MetaCacheEntry<HiveExternalMetaCache.PartitionCacheKey,
HivePartition> partitionCache,
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCacheTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCacheTest.java
index e4e7f6a9406..dd99b578c4a 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCacheTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCacheTest.java
@@ -18,6 +18,7 @@
package org.apache.doris.datasource.maxcompute;
import org.apache.doris.catalog.Type;
+import org.apache.doris.common.Config;
import org.apache.doris.datasource.NameMapping;
import org.apache.doris.datasource.SchemaCacheKey;
import org.apache.doris.datasource.SchemaCacheValue;
@@ -108,4 +109,31 @@ public class MaxComputeExternalMetaCacheTest {
executor.shutdownNow();
}
}
+
+ @Test
+ public void testPartitionValuesDefaultSpecUsesTableLevelCapacity() {
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ long originalPartitionCapacity = Config.max_hive_partition_cache_num;
+ long originalPartitionTableCapacity =
Config.max_hive_partition_table_cache_num;
+ long originalRefreshTime = Config.external_cache_refresh_time_minutes;
+ try {
+ Config.max_hive_partition_cache_num = 100L;
+ Config.max_hive_partition_table_cache_num = 20L;
+ Config.external_cache_refresh_time_minutes = 3L;
+
+ MaxComputeExternalMetaCache cache = new
MaxComputeExternalMetaCache(executor);
+ long catalogId = 1L;
+ cache.initCatalog(catalogId, Collections.emptyMap());
+
+ MetaCacheEntryStats partitionValuesStats = cache.stats(catalogId)
+ .get(MaxComputeExternalMetaCache.ENTRY_PARTITION_VALUES);
+ Assert.assertEquals(20L, partitionValuesStats.getCapacity());
+ Assert.assertEquals(180L, partitionValuesStats.getTtlSecond());
+ } finally {
+ Config.max_hive_partition_cache_num = originalPartitionCapacity;
+ Config.max_hive_partition_table_cache_num =
originalPartitionTableCapacity;
+ Config.external_cache_refresh_time_minutes = originalRefreshTime;
+ executor.shutdownNow();
+ }
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java
index 39ab13fcb7b..290eed874ed 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java
@@ -166,6 +166,29 @@ public class MetaCacheEntryTest {
}
}
+ @Test
+ public void testStatsSnapshotContainsEvictionRate() throws Exception {
+ ExecutorService refreshExecutor = Executors.newSingleThreadExecutor();
+ try {
+ CacheSpec cacheSpec = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL,
1L);
+ MetaCacheEntry<String, Integer> entry = new MetaCacheEntry<>(
+ "test",
+ String::length,
+ cacheSpec,
+ refreshExecutor,
+ false);
+
+ Assert.assertEquals(0D, entry.stats().getEvictionRate(), 0D);
+ Assert.assertEquals(Integer.valueOf(1), entry.get("a"));
+ Assert.assertEquals(Integer.valueOf(2), entry.get("bb"));
+ extractLoadingCache(entry).cleanUp();
+ Assert.assertEquals(1L, entry.stats().getEvictionCount());
+ Assert.assertEquals(0.5D, entry.stats().getEvictionRate(), 0D);
+ } finally {
+ refreshExecutor.shutdownNow();
+ }
+ }
+
@Test
public void testContextualOnlyEntryRejectsDefaultGet() {
ExecutorService refreshExecutor = Executors.newSingleThreadExecutor();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
