This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 171340ad38e HIVE-29046: Avoid unecessary perf overhead from handling
deprecated properties in MetastoreConf (#5897)
171340ad38e is described below
commit 171340ad38ee8d082f6992a416c5aad63377d708
Author: Stamatis Zampetakis <zabe...@gmail.com>
AuthorDate: Fri Jul 4 15:16:52 2025 +0200
HIVE-29046: Avoid unecessary perf overhead from handling deprecated
properties in MetastoreConf (#5897)
Looping through ~400 ConfVars enumeration entries on every creation of a
new Configuration entry is resource wasteful and unecessary.
The deprecated properties are static and known at compile time so there is
no reason to search and register them on every creation of a new Configuration
instance.
In addition, the presence of deprecation information inside the ConfVars
enumeration leads to boilerplate code that is not really necessary.
1. Move deprecated property registation in the existing static block
2. Drop constructors and variables related to deprecated keys in ConfVars
3. Drop DEPRECATED_TEST_ENTRY that is only useful for testing purposes and
adapt existing tests
---
.../hadoop/hive/metastore/conf/MetastoreConf.java | 60 ++++------------------
.../hive/metastore/conf/TestMetastoreConf.java | 12 ++---
2 files changed, 15 insertions(+), 57 deletions(-)
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 3627517b441..e8fd6190040 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -128,6 +128,13 @@ public class MetastoreConf {
keyToVars.put(var.varname, var);
keyToVars.put(var.hiveName, var);
}
+ Configuration.addDeprecations(new Configuration.DeprecationDelta[] {
+ new
Configuration.DeprecationDelta("metastore.compactor.history.retention.attempted",
+ ConfVars.COMPACTOR_HISTORY_RETENTION_DID_NOT_INITIATE.varname),
+ new
Configuration.DeprecationDelta("hive.compactor.history.retention.attempted",
+ ConfVars.COMPACTOR_HISTORY_RETENTION_DID_NOT_INITIATE.varname),
+ new
Configuration.DeprecationDelta("datanucleus.plugin.pluginRegistryBundleCheck",
+ ConfVars.DATANUCLEUS_PLUGIN_REGISTRY_BUNDLE_CHECK.varname)});
}
@VisibleForTesting
@@ -421,9 +428,7 @@ public enum ConfVars {
COMPACTOR_HISTORY_RETENTION_DID_NOT_INITIATE("metastore.compactor.history.retention.did.not.initiate",
"hive.compactor.history.retention.did.not.initiate", 2,
new RangeValidator(0, 100), "Determines how many compaction records in
state " +
- "'did not initiate' will be retained in compaction history for a given
table/partition.",
- // deprecated keys:
- "metastore.compactor.history.retention.attempted",
"hive.compactor.history.retention.attempted"),
+ "'did not initiate' will be retained in compaction history for a given
table/partition."),
COMPACTOR_HISTORY_RETENTION_FAILED("metastore.compactor.history.retention.failed",
"hive.compactor.history.retention.failed", 3,
new RangeValidator(0, 100), "Determines how many failed compaction
records will be " +
@@ -748,8 +753,7 @@ public enum ConfVars {
"initializeColumnInfo setting for DataNucleus; set to NONE at least on
Postgres."),
DATANUCLEUS_PLUGIN_REGISTRY_BUNDLE_CHECK("datanucleus.plugin.pluginRegistryBundleCheck".toLowerCase(),
"datanucleus.plugin.pluginRegistryBundleCheck", "LOG", true,
- "Defines what happens when plugin bundles are found and are duplicated
[EXCEPTION|LOG|NONE]",
- "datanucleus.plugin.pluginRegistryBundleCheck", null),
+ "Defines what happens when plugin bundles are found and are duplicated
[EXCEPTION|LOG|NONE]"),
DATANUCLEUS_TRANSACTION_ISOLATION("datanucleus.transactionIsolation",
"datanucleus.transactionIsolation", "read-committed",
"Default transaction isolation level for identity generation."),
@@ -1945,8 +1949,6 @@ public enum ConfVars {
LONG_TEST_ENTRY("test.long", "hive.test.long", 42, "comment"),
DOUBLE_TEST_ENTRY("test.double", "hive.test.double", Math.PI, "comment"),
TIME_TEST_ENTRY("test.time", "hive.test.time", 1, TimeUnit.SECONDS,
"comment"),
- DEPRECATED_TEST_ENTRY("test.deprecated", "hive.test.deprecated", 0, new
RangeValidator(0, 3), "comment",
- "this.is.the.metastore.deprecated.name",
"this.is.the.hive.deprecated.name"),
TIME_VALIDATOR_ENTRY_INCLUSIVE("test.time.validator.inclusive",
"hive.test.time.validator.inclusive", 1,
TimeUnit.SECONDS,
new TimeValidator(TimeUnit.MILLISECONDS, 500L, true, 1500L, true),
"comment"),
@@ -1962,8 +1964,6 @@ public enum ConfVars {
private final Validator validator;
private final boolean caseSensitive;
private final String description;
- private String deprecatedName = null;
- private String hiveDeprecatedName = null;
ConfVars(String varname, String hiveName, String defaultVal, String
description) {
this.varname = varname;
@@ -1994,18 +1994,6 @@ public enum ConfVars {
this.description = description;
}
- ConfVars(String varname, String hiveName, String defaultVal, boolean
caseSensitive,
- String description, String deprecatedName, String hiveDeprecatedName) {
- this.varname = varname;
- this.hiveName = hiveName;
- this.defaultVal = defaultVal;
- validator = null;
- this.caseSensitive = caseSensitive;
- this.description = description;
- this.deprecatedName = deprecatedName;
- this.hiveDeprecatedName = hiveDeprecatedName;
- }
-
ConfVars(String varname, String hiveName, long defaultVal, String
description) {
this.varname = varname;
this.hiveName = hiveName;
@@ -2025,18 +2013,6 @@ public enum ConfVars {
this.description = description;
}
- ConfVars(String varname, String hiveName, long defaultVal, Validator
validator,
- String description, String deprecatedName, String hiveDeprecatedName) {
- this.varname = varname;
- this.hiveName = hiveName;
- this.defaultVal = defaultVal;
- this.validator = validator;
- caseSensitive = false;
- this.description = description;
- this.deprecatedName = deprecatedName;
- this.hiveDeprecatedName = hiveDeprecatedName;
- }
-
ConfVars(String varname, String hiveName, boolean defaultVal, String
description) {
this.varname = varname;
this.hiveName = hiveName;
@@ -2242,24 +2218,6 @@ public static Configuration
newMetastoreConf(Configuration conf) {
LOG.isDebugEnabled()) {
LOG.debug(dumpConfig(conf));
}
-
- /*
- Add deprecated config names to configuration.
- The parameters for Configuration.addDeprecation are (oldKey, newKey) and
it is assumed that the config is set via
- newKey and the value is retrieved via oldKey.
- However in this case we assume the value is set with the deprecated key
(oldKey) in some config file and we
- retrieve it in the code via the new key. So the parameter order we use
here is: (newKey, deprecatedKey).
- We do this with the HiveConf configs as well.
- */
- for (ConfVars var : ConfVars.values()) {
- if (var.deprecatedName != null) {
- Configuration.addDeprecation(var.getVarname(), var.deprecatedName);
- }
- if (var.hiveDeprecatedName != null) {
- Configuration.addDeprecation(var.getHiveName(),
var.hiveDeprecatedName);
- }
- }
-
return conf;
}
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java
index b5208c7fe75..03e5351baad 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java
@@ -321,26 +321,26 @@ public void testDeprecatedConfigs() throws IOException {
// set with deprecated key
createConfFile("metastore-site.xml", false, "METASTORE_CONF_DIR", instaMap(
"hive.test.str", "hivedefault",
- "this.is.the.metastore.deprecated.name", "1" // default is 0
+ "metastore.compactor.history.retention.attempted", "0" // default is 2
));
conf = MetastoreConf.newMetastoreConf();
- Assert.assertEquals(1, MetastoreConf.getIntVar(conf,
ConfVars.DEPRECATED_TEST_ENTRY));
+ Assert.assertEquals(0, MetastoreConf.getIntVar(conf,
ConfVars.COMPACTOR_HISTORY_RETENTION_DID_NOT_INITIATE));
// set with hive (HiveConf) deprecated key
createConfFile("metastore-site.xml", false, "METASTORE_CONF_DIR", instaMap(
"hive.test.str", "hivedefault",
- "this.is.the.hive.deprecated.name", "2" // default is 0
+ "hive.compactor.history.retention.attempted", "1" // default is 2
));
conf = MetastoreConf.newMetastoreConf();
- Assert.assertEquals(2, MetastoreConf.getIntVar(conf,
ConfVars.DEPRECATED_TEST_ENTRY));
+ Assert.assertEquals(1, MetastoreConf.getIntVar(conf,
ConfVars.COMPACTOR_HISTORY_RETENTION_DID_NOT_INITIATE));
// set with normal key
createConfFile("metastore-site.xml", false, "METASTORE_CONF_DIR", instaMap(
"hive.test.str", "hivedefault",
- "test.deprecated", "3" // default is 0
+ "metastore.compactor.history.retention.did.not.initiate", "3" //
default is 2
));
conf = MetastoreConf.newMetastoreConf();
- Assert.assertEquals(3, MetastoreConf.getIntVar(conf,
ConfVars.DEPRECATED_TEST_ENTRY));
+ Assert.assertEquals(3, MetastoreConf.getIntVar(conf,
ConfVars.COMPACTOR_HISTORY_RETENTION_DID_NOT_INITIATE));
}
@Test
