This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f84cfcfb9bc [opt](config) support dynamically change hms timeout
(#59051)
f84cfcfb9bc is described below
commit f84cfcfb9bc8d6647b44c6d36abe832ae3e59317
Author: camby <[email protected]>
AuthorDate: Mon Dec 22 14:20:58 2025 +0800
[opt](config) support dynamically change hms timeout (#59051)
### What problem does this PR solve?
Now we use FE config `hive_metastore_client_timeout_second` to force
reset
`hive.metastore.client.socket.timeout`, but there are some problem:
1. config `hive_metastore_client_timeout_second` is global, can not set
for each catalog;
2. config `hive_metastore_client_timeout_second` is Mutable, but it will
not take effect because of `ThriftHMSCachedClient`；
How to improve:
keep config `hive_metastore_client_timeout_second` as the default
timeout, while user do not override
`hive.metastore.client.socket.timeout`, and make the config immutable.
---
.../src/main/java/org/apache/doris/common/Config.java | 2 +-
.../doris/datasource/hive/ThriftHMSCachedClient.java | 5 -----
.../datasource/property/metastore/HMSBaseProperties.java | 8 ++++++--
.../datasource/property/metastore/HMSPropertiesTest.java | 15 +++++++++++++++
4 files changed, 22 insertions(+), 8 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 5faca24e2f0..2329b703fd9 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2246,7 +2246,7 @@ public class Config extends ConfigBase {
* The default connection timeout for hive metastore.
* hive.metastore.client.socket.timeout
*/
- @ConfField(mutable = true, masterOnly = false)
+ @ConfField(mutable = false, masterOnly = false)
public static long hive_metastore_client_timeout_second = 10;
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java
index 1b763969afa..840ad765587 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
@@ -98,10 +97,6 @@ public class ThriftHMSCachedClient implements
HMSCachedClient {
public ThriftHMSCachedClient(HiveConf hiveConf, int poolSize,
ExecutionAuthenticator executionAuthenticator) {
Preconditions.checkArgument(poolSize > 0, poolSize);
- if (hiveConf != null) {
- HiveConf.setVar(hiveConf, ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT,
-
String.valueOf(Config.hive_metastore_client_timeout_second));
- }
this.hiveConf = hiveConf;
this.poolSize = poolSize;
this.isClosed = false;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSBaseProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSBaseProperties.java
index 9121d2148e6..9056aae6ee7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSBaseProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSBaseProperties.java
@@ -30,6 +30,7 @@ import com.google.common.base.Strings;
import lombok.Getter;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import java.util.HashMap;
import java.util.Map;
@@ -198,8 +199,11 @@ public class HMSBaseProperties {
if (StringUtils.isNotBlank(hmsUserName)) {
hiveConf.set(AuthenticationConfig.HADOOP_USER_NAME, hmsUserName);
}
- HiveConf.setVar(hiveConf,
HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT,
- String.valueOf(Config.hive_metastore_client_timeout_second));
+ if
(!userOverriddenHiveConfig.containsKey(ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT.toString()))
{
+ // use Config.hive_metastore_client_timeout_second as default
timeout
+ HiveConf.setVar(hiveConf,
HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT,
+
String.valueOf(Config.hive_metastore_client_timeout_second));
+ }
initHadoopAuthenticator();
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSPropertiesTest.java
index 88ca5fa7c95..6d240f65729 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSPropertiesTest.java
@@ -134,4 +134,19 @@ public class HMSPropertiesTest {
HiveHMSProperties hmsProperties = getHMSProperties(params);
Assertions.assertNotNull(hmsProperties);
}
+
+ @Test
+ public void testHmsTimeoutParams() throws UserException {
+ Map<String, String> params = createBaseParams();
+ // case1: normal case, use Config.hive_metastore_client_timeout_second
as default
+ HiveHMSProperties hmsProperties = getHMSProperties(params);
+ HiveConf hiveConf = hmsProperties.getHiveConf();
+
Assertions.assertEquals(String.valueOf(Config.hive_metastore_client_timeout_second),
+ hiveConf.get("hive.metastore.client.socket.timeout", null));
+
+ params.put("hive.metastore.client.socket.timeout", "123");
+ hmsProperties = getHMSProperties(params);
+ hiveConf = hmsProperties.getHiveConf();
+ Assertions.assertEquals("123",
hiveConf.get("hive.metastore.client.socket.timeout", null));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
