This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-1.2-lts in repository https://gitbox.apache.org/repos/asf/doris.git
commit 80f1508065ae6de893a65058ad007e8274d2fba9 Author: Jibing-Li <[email protected]> AuthorDate: Wed May 10 10:55:33 2023 +0800 [Fix](multi catalog)Return all partition values while reading hive table. (#19434) Return all partition values while reading hive table. Add a config item for the max value of hive table to partition list cache. Default value is 100. --- fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 8 ++++++++ .../java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java | 2 +- .../apache/doris/datasource/hive/PooledHiveMetaStoreClient.java | 3 ++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 0466e31f1d..9e3c3aac3e 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1900,6 +1900,14 @@ public class Config extends ConfigBase { @ConfField(mutable = false, masterOnly = false) public static long max_hive_partition_cache_num = 100000; + @ConfField(mutable = false, masterOnly = false, description = {"Hive表到分区名列表缓存的最大数量。", + "Max cache number of hive table to partition names list."}) + public static long max_hive_table_catch_num = 1000; + + @ConfField(mutable = false, masterOnly = false, description = {"获取Hive分区值时候的最大返回数量，-1代表没有限制。", + "Max number of hive partition values to return while list partitions, -1 means no limitation."}) + public static short max_hive_list_partition_num = -1; + /** * Max cache loader thread-pool size. * Max thread pool size for loading external meta cache diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index a2fb14de39..95f0b957c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -99,7 +99,7 @@ public class HiveMetaStoreCache { } private void init(Executor executor) { - partitionValuesCache = CacheBuilder.newBuilder().maximumSize(Config.max_hive_partition_cache_num) + partitionValuesCache = CacheBuilder.newBuilder().maximumSize(Config.max_hive_table_catch_num) .expireAfterAccess(Config.external_cache_expire_time_minutes_after_access, TimeUnit.MINUTES) .build(CacheLoader.asyncReloading( new CacheLoader<PartitionValueCacheKey, HivePartitionValues>() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java index 0982da418c..236aacf298 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java @@ -53,7 +53,8 @@ public class PooledHiveMetaStoreClient { private static final Logger LOG = LogManager.getLogger(PooledHiveMetaStoreClient.class); private static final HiveMetaHookLoader DUMMY_HOOK_LOADER = t -> null; - private static final short MAX_LIST_PARTITION_NUM = 10000; + // -1 means no limit on the partitions returned. + private static final short MAX_LIST_PARTITION_NUM = Config.max_hive_list_partition_num; private Queue<CachedClient> clientPool = new LinkedList<>(); private final int poolSize; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
