This is an automated email from the ASF dual-hosted git repository.
dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 55dff6a525c HIVE-29396: Drop the property
hive.metastore.try.direct.sql.ddl (#6265)
55dff6a525c is described below
commit 55dff6a525c0d3cbe13f25dbae460d25d7c164d6
Author: dengzh <[email protected]>
AuthorDate: Sun Jan 18 10:32:30 2026 +0800
HIVE-29396: Drop the property hive.metastore.try.direct.sql.ddl (#6265)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 118 ---------------------
.../sqlstd/TestSQLStdHiveAccessControllerHS2.java | 5 +-
.../clientpositive/alter_table_column_stats.q | 4 +-
.../hadoop/hive/metastore/conf/MetastoreConf.java | 8 --
.../apache/hadoop/hive/metastore/ObjectStore.java | 6 +-
...GetPartitionsUsingProjectionAndFilterSpecs.java | 1 -
.../metastore/TestHiveMetaStoreGetMetaConf.java | 9 --
.../client/TestTablesCreateDropAlterTruncate.java | 3 -
8 files changed, 6 insertions(+), 148 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 4a4a470c0d1..75b95d956e5 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -97,7 +97,6 @@ public class HiveConf extends Configuration {
private static byte[] confVarByteArray = null;
private static final Map<String, ConfVars> vars = new HashMap<String,
ConfVars>();
- private static final Map<String, ConfVars> metaConfs = new HashMap<String,
ConfVars>();
private final List<String> restrictList = new ArrayList<String>();
private final Set<String> hiddenSet = new HashSet<String>();
private final Set<String> lockedSet = new HashSet<>();
@@ -252,118 +251,11 @@ private static URL checkConfigFile(File f) {
}
}
-
-
-
@InterfaceAudience.Private
public static final String PREFIX_LLAP = "llap.";
@InterfaceAudience.Private
public static final String PREFIX_HIVE_LLAP = "hive.llap.";
- /**
- * Metastore related options that the db is initialized against. When a conf
- * var in this is list is changed, the metastore instance for the CLI will
- * be recreated so that the change will take effect.
- */
- public static final HiveConf.ConfVars[] metaVars = {
- HiveConf.ConfVars.METASTORE_WAREHOUSE,
- HiveConf.ConfVars.REPL_DIR,
- HiveConf.ConfVars.METASTORE_URIS,
- HiveConf.ConfVars.METASTORE_SELECTION,
- HiveConf.ConfVars.METASTORE_SERVER_PORT,
- HiveConf.ConfVars.METASTORE_THRIFT_CONNECTION_RETRIES,
- HiveConf.ConfVars.METASTORE_THRIFT_FAILURE_RETRIES,
- HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY,
- HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT,
- HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME,
- HiveConf.ConfVars.METASTORE_PWD,
- HiveConf.ConfVars.METASTORE_CONNECT_URL_HOOK,
- HiveConf.ConfVars.METASTORE_CONNECT_URL_KEY,
- HiveConf.ConfVars.METASTORE_SERVER_MIN_THREADS,
- HiveConf.ConfVars.METASTORE_SERVER_MAX_THREADS,
- HiveConf.ConfVars.METASTORE_INT_ORIGINAL,
- HiveConf.ConfVars.METASTORE_INT_ARCHIVED,
- HiveConf.ConfVars.METASTORE_INT_EXTRACTED,
- HiveConf.ConfVars.METASTORE_KERBEROS_KEYTAB_FILE,
- HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL,
- HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL,
- HiveConf.ConfVars.METASTORE_TOKEN_SIGNATURE,
- HiveConf.ConfVars.METASTORE_CACHE_PINOBJTYPES,
- HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE,
- HiveConf.ConfVars.METASTORE_VALIDATE_TABLES,
- HiveConf.ConfVars.METASTORE_DATANUCLEUS_INIT_COL_INFO,
- HiveConf.ConfVars.METASTORE_VALIDATE_COLUMNS,
- HiveConf.ConfVars.METASTORE_VALIDATE_CONSTRAINTS,
- HiveConf.ConfVars.METASTORE_STORE_MANAGER_TYPE,
- HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL,
- HiveConf.ConfVars.METASTORE_TRANSACTION_ISOLATION,
- HiveConf.ConfVars.METASTORE_CACHE_LEVEL2,
- HiveConf.ConfVars.METASTORE_CACHE_LEVEL2_TYPE,
- HiveConf.ConfVars.METASTORE_IDENTIFIER_FACTORY,
- HiveConf.ConfVars.METASTORE_PLUGIN_REGISTRY_BUNDLE_CHECK,
- HiveConf.ConfVars.METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS,
- HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX,
- HiveConf.ConfVars.METASTORE_EVENT_LISTENERS,
- HiveConf.ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS,
- HiveConf.ConfVars.METASTORE_EVENT_CLEAN_FREQ,
- HiveConf.ConfVars.METASTORE_EVENT_EXPIRY_DURATION,
- HiveConf.ConfVars.METASTORE_EVENT_MESSAGE_FACTORY,
- HiveConf.ConfVars.METASTORE_FILTER_HOOK,
- HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
- HiveConf.ConfVars.METASTORE_END_FUNCTION_LISTENERS,
- HiveConf.ConfVars.METASTORE_PART_INHERIT_TBL_PROPS,
- HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_OBJECTS_MAX,
- HiveConf.ConfVars.METASTORE_INIT_HOOKS,
- HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS,
- HiveConf.ConfVars.HMS_HANDLER_ATTEMPTS,
- HiveConf.ConfVars.HMS_HANDLER_INTERVAL,
- HiveConf.ConfVars.HMS_HANDLER_FORCE_RELOAD_CONF,
- HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN,
- HiveConf.ConfVars.METASTORE_ORM_RETRIEVE_MAPNULLS_AS_EMPTY_STRINGS,
- HiveConf.ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES,
- HiveConf.ConfVars.USERS_IN_ADMIN_ROLE,
- HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
- HiveConf.ConfVars.HIVE_TXN_MANAGER,
- HiveConf.ConfVars.HIVE_TXN_TIMEOUT,
- HiveConf.ConfVars.HIVE_TXN_OPERATIONAL_PROPERTIES,
- HiveConf.ConfVars.HIVE_TXN_HEARTBEAT_THREADPOOL_SIZE,
- HiveConf.ConfVars.HIVE_TXN_MAX_OPEN_BATCH,
- HiveConf.ConfVars.HIVE_TXN_RETRYABLE_SQLEX_REGEX,
- HiveConf.ConfVars.HIVE_METASTORE_STATS_NDV_TUNER,
- HiveConf.ConfVars.HIVE_METASTORE_STATS_NDV_DENSITY_FUNCTION,
- HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_ENABLED,
- HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_SIZE,
- HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_PARTITIONS,
- HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_FPP,
- HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_VARIANCE,
- HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_TTL,
- HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_WRITER_WAIT,
- HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_READER_WAIT,
- HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_FULL,
- HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_CLEAN_UNTIL,
- HiveConf.ConfVars.METASTORE_FASTPATH,
- HiveConf.ConfVars.METASTORE_HBASE_FILE_METADATA_THREADS,
- HiveConf.ConfVars.METASTORE_WM_DEFAULT_POOL_SIZE
- };
-
- /**
- * User configurable Metastore vars
- */
- static final HiveConf.ConfVars[] metaConfVars = {
- HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL,
- HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL_DDL,
- HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT,
- HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN,
- HiveConf.ConfVars.METASTORE_CAPABILITY_CHECK,
- HiveConf.ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES
- };
-
- static {
- for (ConfVars confVar : metaConfVars) {
- metaConfs.put(confVar.varname, confVar);
- }
- }
-
public static final String HIVE_LLAP_DAEMON_SERVICE_PRINCIPAL_NAME =
"hive.llap.daemon.service.principal";
public static final String
HIVE_SERVER2_AUTHENTICATION_LDAP_USERMEMBERSHIPKEY_NAME =
"hive.server2.authentication.ldap.userMembershipKey";
@@ -6466,10 +6358,6 @@ public static ConfVars getConfVars(String name) {
return vars.get(name);
}
- public static ConfVars getMetaConf(String name) {
- return metaConfs.get(name);
- }
-
public String getVar(ConfVars var) {
return getVar(this, var);
}
@@ -6665,12 +6553,6 @@ private void initialize(Class<?> cls) {
// Overlay the values of any system properties and manual overrides
applySystemProperties();
- if ((this.get("hive.metastore.ds.retry.attempts") != null) ||
- this.get("hive.metastore.ds.retry.interval") != null) {
- LOG.warn("DEPRECATED: hive.metastore.ds.retry.* no longer has any
effect. " +
- "Use hive.hmshandler.retry.* instead");
- }
-
// if the running class was loaded directly (through eclipse) rather than
through a
// jar then this would be needed
if (hiveJar == null) {
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java
b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java
index 1fc3663e75b..0d61eaf2573 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java
@@ -27,6 +27,7 @@
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.ql.security.HadoopDefaultAuthenticator;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.DisallowTransformHook;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException;
@@ -126,8 +127,8 @@ private void verifyParamSettability(List<String>
settableParams, HiveConf proces
assertConfModificationException(processedConf, "dummy.param");
// does not make sense to have any of the metastore config variables to be
// modifiable
- for (ConfVars metaVar : HiveConf.metaVars) {
- assertConfModificationException(processedConf, metaVar.varname);
+ for (MetastoreConf.ConfVars metaVar : MetastoreConf.metaVars) {
+ assertConfModificationException(processedConf, metaVar.getHiveName());
}
}
diff --git a/ql/src/test/queries/clientpositive/alter_table_column_stats.q
b/ql/src/test/queries/clientpositive/alter_table_column_stats.q
index cd699b32dc3..3440758c500 100644
--- a/ql/src/test/queries/clientpositive/alter_table_column_stats.q
+++ b/ql/src/test/queries/clientpositive/alter_table_column_stats.q
@@ -266,8 +266,8 @@ use default;
drop database statsdb1;
drop database statsdb2;
--- Test for external tables with metastore.try.direct.sql.ddl as false
-set metaconf:metastore.try.direct.sql.ddl=false;
+-- Test for external tables with metastore.try.direct.sql as false
+set metaconf:metastore.try.direct.sql=false;
drop database if exists statsdb1;
create database statsdb1;
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 85b37d67889..16ccf508a43 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -250,7 +250,6 @@ public String toString() {
ConfVars.METASTORE_PARTITIONS_PARAMETERS_EXCLUDE_PATTERN,
// Add metaConfVars here as well
ConfVars.TRY_DIRECT_SQL,
- ConfVars.TRY_DIRECT_SQL_DDL,
ConfVars.CLIENT_SOCKET_TIMEOUT,
ConfVars.PARTITION_NAME_WHITELIST_PATTERN,
ConfVars.PARTITION_ORDER_EXPR,
@@ -264,7 +263,6 @@ public String toString() {
*/
public static final MetastoreConf.ConfVars[] metaConfVars = {
ConfVars.TRY_DIRECT_SQL,
- ConfVars.TRY_DIRECT_SQL_DDL,
ConfVars.CLIENT_SOCKET_TIMEOUT,
ConfVars.PARTITION_NAME_WHITELIST_PATTERN,
ConfVars.PARTITION_ORDER_EXPR,
@@ -1663,12 +1661,6 @@ public enum ConfVars {
"work for all queries on your datastore. If all SQL queries fail
(for example, your\n" +
"metastore is backed by MongoDB), you might want to disable this
to save the\n" +
"try-and-fall-back cost."),
- TRY_DIRECT_SQL_DDL("metastore.try.direct.sql.ddl",
"hive.metastore.try.direct.sql.ddl", true,
- "Same as hive.metastore.try.direct.sql, for read statements within a
transaction that\n" +
- "modifies metastore data. Due to non-standard behavior in
Postgres, if a direct SQL\n" +
- "select query has incorrect syntax or something similar inside a
transaction, the\n" +
- "entire transaction will fail and fall-back to DataNucleus will
not be possible. You\n" +
- "should disable the usage of direct SQL inside transactions if
that happens in your case."),
TXN_MAX_OPEN_BATCH("metastore.txn.max.open.batch",
"hive.txn.max.open.batch", 1000,
"Maximum number of transactions that can be fetched in one call to
open_txns().\n" +
"This controls how many transactions streaming agents such as
Flume or Storm open\n" +
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index f6954bf9cc6..3820c8c500e 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -4203,11 +4203,7 @@ public GetHelper(String catalogName, String dbName,
String tblName,
this.doTrace = LOG.isDebugEnabled();
this.isInTxn = isActiveTransaction();
- // SQL usage inside a larger transaction (e.g. droptable) may not be
desirable because
- // some databases (e.g. Postgres) abort the entire transaction when any
query fails, so
- // the fallback from failed SQL to JDO is not possible.
- boolean isConfigEnabled = MetastoreConf.getBoolVar(getConf(),
ConfVars.TRY_DIRECT_SQL)
- && (MetastoreConf.getBoolVar(getConf(), ConfVars.TRY_DIRECT_SQL_DDL)
|| !isInTxn);
+ boolean isConfigEnabled = MetastoreConf.getBoolVar(getConf(),
ConfVars.TRY_DIRECT_SQL);
if (isConfigEnabled && directSql == null) {
directSql = new MetaStoreDirectSql(pm, getConf(), "");
}
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestGetPartitionsUsingProjectionAndFilterSpecs.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestGetPartitionsUsingProjectionAndFilterSpecs.java
index 7a37f6032b4..f284849b73f 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestGetPartitionsUsingProjectionAndFilterSpecs.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestGetPartitionsUsingProjectionAndFilterSpecs.java
@@ -113,7 +113,6 @@ public void setup() throws TException {
tbl = client.getTable(dbName, tblName);
// set directSQL to true explicitly
client.setMetaConf(ConfVars.TRY_DIRECT_SQL.getVarname(), "true");
- client.setMetaConf(ConfVars.TRY_DIRECT_SQL_DDL.getVarname(), "true");
}
@After
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java
index 5963cb020ab..c8b3d207cdf 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java
@@ -60,7 +60,6 @@ public static void startMetaStoreServer() throws Exception {
Configuration metastoreConf = MetastoreConf.newMetastoreConf();
MetastoreConf.setClass(metastoreConf, ConfVars.EXPRESSION_PROXY_CLASS,
MockPartitionExpressionForMetastore.class,
PartitionExpressionProxy.class);
- MetastoreConf.setBoolVar(metastoreConf, ConfVars.TRY_DIRECT_SQL_DDL,
false);
MetaStoreTestUtils.setConfForStandloneMode(metastoreConf);
int msPort = MetaStoreServerUtils.startMetaStore(metastoreConf);
conf = MetastoreConf.newMetastoreConf();
@@ -97,14 +96,6 @@ public void testGetMetaConfDefaultEmptyString() throws
TException {
assertEquals(expected, actual);
}
- @Test
- public void testGetMetaConfOverridden() throws TException {
- ConfVars metaConfVar = ConfVars.TRY_DIRECT_SQL_DDL;
- String expected = "false";
- String actual = hmsc.getMetaConf(metaConfVar.toString());
- assertEquals(expected, actual);
- }
-
@Test
public void testGetMetaConfUnknownPreperty() throws TException {
String unknownPropertyName = "hive.meta.foo.bar";
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java
index 3f7ec97b79e..bf09765634e 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java
@@ -1198,7 +1198,6 @@ public void testAlterTableAlreadyExists() throws
Exception {
@Test
public void testAlterTableExpectedPropertyMatch() throws Exception {
Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(),
ConfVars.TRY_DIRECT_SQL));
- Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(),
ConfVars.TRY_DIRECT_SQL_DDL));
Table originalTable = testTables[0];
EnvironmentContext context = new EnvironmentContext();
@@ -1213,7 +1212,6 @@ public void testAlterTableExpectedPropertyMatch() throws
Exception {
@Test(expected = MetaException.class)
public void testAlterTableExpectedPropertyDifferent() throws Exception {
Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(),
ConfVars.TRY_DIRECT_SQL));
- Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(),
ConfVars.TRY_DIRECT_SQL_DDL));
Table originalTable = testTables[0];
EnvironmentContext context = new EnvironmentContext();
@@ -1234,7 +1232,6 @@ public void testAlterTableExpectedPropertyDifferent()
throws Exception {
@Test
public void testAlterTableExpectedPropertyConcurrent() throws Exception {
Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(),
ConfVars.TRY_DIRECT_SQL));
- Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(),
ConfVars.TRY_DIRECT_SQL_DDL));
Table originalTable = testTables[0];
originalTable.getParameters().put("snapshot", "0");
