This is an automated email from the ASF dual-hosted git repository.
dengzhhu653 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 9f387afc25e HIVE-29520: Add configuration to enable/disable ACID
functionality in Hive Metastore (#6524)
9f387afc25e is described below
commit 9f387afc25ef50ea83423c8ae7ac725a07259594
Author: Venu Reddy <[email protected]>
AuthorDate: Tue Jun 9 08:18:42 2026 +0530
HIVE-29520: Add configuration to enable/disable ACID functionality in Hive
Metastore (#6524)
---
.../metastore/TestHiveMetastoreTransformer.java | 7 +-
.../hadoop/hive/ql/txn/compactor/Initiator.java | 2 +
.../hadoop/hive/metastore/conf/MetastoreConf.java | 9 +-
.../MaterializationsRebuildLockCleanerTask.java | 3 +-
.../metastore/MetastoreDefaultTransformer.java | 23 +++-
.../hive/metastore/leader/CompactorTasks.java | 9 +-
.../hive/metastore/leader/HouseKeepingTasks.java | 18 ++-
.../hive/metastore/metrics/AcidMetricLogger.java | 4 +-
.../hive/metastore/metrics/AcidMetricService.java | 3 +-
.../txn/service/AcidHouseKeeperService.java | 3 +-
.../txn/service/AcidOpenTxnsCounterService.java | 3 +-
.../txn/service/AcidTxnCleanerService.java | 3 +-
.../txn/service/CompactionHouseKeeperService.java | 13 ++-
.../hadoop/hive/metastore/TestNoAcidSupport.java | 127 +++++++++++++++++++++
14 files changed, 193 insertions(+), 34 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
index c3c174bcc34..1dc8b1776dd 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
@@ -25,6 +25,7 @@
import java.util.List;
import java.util.Map;
import java.util.Locale;
+import java.util.Optional;
import org.apache.hadoop.hive.metastore.api.GetPartitionsByNamesRequest;
import org.apache.hadoop.hive.metastore.client.builder.GetTablesRequestBuilder;
@@ -821,6 +822,7 @@ public void testGetTablesExt() throws Exception {
count = 300;
tProps.put("TBLNAME", "test_limit");
tProps.put("TABLECOUNT", count);
+ tProps.remove("CAPABILITIES"); // CAPABILITIES are already appended to
PROPERTIES
tables = createTables(tProps);
assertEquals("Unexpected number of tables created", count,
tables.size());
@@ -934,7 +936,8 @@ public void testGetPartitionsByNames() throws Exception {
properties.append("transactional_properties=insert_only");
tProps.put("TBLNAME", tblName);
tProps.put("PROPERTIES", properties.toString());
- setHMSClient("createTable", new String[]
{"HIVEMANAGEDINSERTWRITE,HIVEFULLACIDWRITE"});
+ tProps.put("TBLTYPE", type);
+ setHMSClient("createTable", new String[] {"HIVEMANAGEDINSERTWRITE",
"HIVEFULLACIDWRITE"});
table = createTableWithCapabilities(tProps);
resetHMSClient();
@@ -1772,7 +1775,7 @@ private List<String> createTables(Map<String, Object>
props) throws Exception {
String tblName = (String)props.get("TBLNAME");
List<String> caps = (List<String>)props.get("CAPABILITIES");
StringBuilder table_params = new StringBuilder();
- table_params.append((String)props.get("PROPERTIES"));
+
Optional.ofNullable(props.get("PROPERTIES")).ifPresent(table_params::append);
if (caps != null)
table_params.append(CAPABILITIES_KEY).append("=").append(String.join(",",
caps));
props.put("PROPERTIES", table_params.toString());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
index 5dc40b2ba9f..1e5fe45f682 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
@@ -213,8 +213,10 @@ public void init(AtomicBoolean stop) throws Exception {
checkInterval =
conf.getTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_CHECK_INTERVAL,
TimeUnit.MILLISECONDS);
metricsEnabled = MetastoreConf.getBoolVar(conf,
MetastoreConf.ConfVars.METRICS_ENABLED) &&
MetastoreConf.getBoolVar(conf,
MetastoreConf.ConfVars.METASTORE_ACIDMETRICS_EXT_ON);
+ boolean isSupportAcid = MetastoreConf.getBoolVar(conf,
MetastoreConf.ConfVars.METASTORE_SUPPORT_ACID);
optimizers = Arrays.stream(MetastoreConf.getTrimmedStringsVar(conf,
MetastoreConf.ConfVars.COMPACTOR_INITIATOR_TABLE_OPTIMIZERS))
+ .filter(e -> isSupportAcid ||
!e.equalsIgnoreCase(MetastoreConf.ACID_TABLE_OPTIMIZER_CLASS))
.map(this::instantiateTableOptimizer).toList();
}
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index e81a9f456d5..e955d1e07d6 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -113,6 +113,10 @@ public class MetastoreConf {
"metastore.authentication.ldap.userMembershipKey";
public static final String METASTORE_RETRYING_HANDLER_CLASS =
"org.apache.hadoop.hive.metastore.RetryingHMSHandler";
+ public static final String ACID_TABLE_OPTIMIZER_CLASS =
+ "org.apache.hadoop.hive.ql.txn.compactor.AcidTableOptimizer";
+ public static final String ICEBERG_TABLE_OPTIMIZER_CLASS =
+ "org.apache.iceberg.mr.hive.compaction.IcebergTableOptimizer";
private static final Map<String, ConfVars> metaConfs = new HashMap<>();
private static volatile URL hiveSiteURL = null;
@@ -663,8 +667,7 @@ public enum ConfVars {
"Enable table caching in the initiator. Currently the cache is cleaned
after each cycle."),
COMPACTOR_INITIATOR_TABLE_OPTIMIZERS("compactor.table.optimizers",
"hive.compactor.table.optimizers",
- "org.apache.hadoop.hive.ql.txn.compactor.AcidTableOptimizer," +
- "org.apache.iceberg.mr.hive.compaction.IcebergTableOptimizer",
+ ACID_TABLE_OPTIMIZER_CLASS + "," + ICEBERG_TABLE_OPTIMIZER_CLASS,
"Comma separated list of table optimizers executed by compaction
Initiator."),
COMPACTOR_WORKER_THREADS("metastore.compactor.worker.threads",
"hive.compactor.worker.threads", 0,
@@ -2012,6 +2015,8 @@ public enum ConfVars {
"The maximum non-native tables allowed per table type during
collecting the summary."),
METADATA_SUMMARY_NONNATIVE_THREADS("hive.metatool.summary.nonnative.threads",
"hive.metatool.summary.nonnative.threads", 20,
"Number of threads to be allocated for MetaToolTaskMetadataSummary for
collecting the non-native table's summary."),
+ METASTORE_SUPPORT_ACID("metastore.support.acid",
"hive.metastore.support.acid", true,
+ "Whether to support acid functionality in Hive metastore server."),
// These are all values that we put here just for testing
STR_TEST_ENTRY("test.str", "hive.test.str", "defaultval", "comment"),
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java
index 10f9721be21..5d1f5dbfe54 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java
@@ -42,7 +42,8 @@ public class MaterializationsRebuildLockCleanerTask
implements MetastoreTaskThre
@Override
public long runFrequency(TimeUnit unit) {
- return MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.TXN_TIMEOUT,
unit) / 2;
+ return MetastoreConf.getBoolVar(conf,
MetastoreConf.ConfVars.METASTORE_SUPPORT_ACID) ?
+ MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.TXN_TIMEOUT,
unit) / 2 : 0;
}
@Override
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
index 468f3b0db5f..e0f66ae7fa1 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
@@ -42,6 +42,7 @@
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
import org.apache.hadoop.hive.metastore.utils.FileUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.slf4j.Logger;
@@ -614,6 +615,14 @@ private static final Path getDefaultPath(IHMSHandler
hmsHandler, Database db, St
}
+ private void validateIfAcidTablePermitted(Table table) throws MetaException {
+ if (!MetastoreConf.getBoolVar(hmsHandler.getConf(),
ConfVars.METASTORE_SUPPORT_ACID) &&
+ TxnUtils.isTransactionalTable(table)) {
+ throw new MetaException("ACID tables are not permitted when the "
+ + ConfVars.METASTORE_SUPPORT_ACID.getHiveName() + " property is set
to false");
+ }
+ }
+
@Override
public Table transformCreateTable(Table table, List<String>
processorCapabilities, String processorId) throws MetaException {
if (!defaultCatalog.equalsIgnoreCase(table.getCatName())) {
@@ -624,9 +633,10 @@ public Table transformCreateTable(Table table,
List<String> processorCapabilitie
Table newTable = new Table(table);
LOG.info("Starting translation for CreateTable for processor " +
processorId + " with " + processorCapabilities
+ " on table " + newTable.getTableName());
- Map<String, String> params = table.getParameters();
+ Map<String, String> params = newTable.getParameters();
if (params == null) {
params = new HashMap<>();
+ newTable.setParameters(params);
}
String tableType = newTable.getTableType();
String dbName = table.getDbName();
@@ -647,7 +657,6 @@ public Table transformCreateTable(Table table, List<String>
processorCapabilitie
params.put(HiveMetaHook.EXTERNAL, "TRUE");
params.put(EXTERNAL_TABLE_PURGE, "TRUE");
params.put(HiveMetaHook.TRANSLATED_TO_EXTERNAL, "TRUE");
- newTable.setParameters(params);
LOG.info("Modified table params are:" + params.toString());
if (getLocation(table) == null) {
@@ -663,6 +672,7 @@ public Table transformCreateTable(Table table, List<String>
processorCapabilitie
// should we check tbl directory existence?
}
} else { // ACID table
+ validateIfAcidTablePermitted(newTable);
// if the property 'EXTERNAL_TABLES_ONLY'='true' is set on the
database, then creating managed/ACID tables are prohibited. See HIVE-25724 for
more details.
if (db.getParameters().containsKey(EXTERNALTABLESONLY) &&
db.getParameters().get(EXTERNALTABLESONLY).equalsIgnoreCase("true")) {
@@ -673,8 +683,8 @@ public Table transformCreateTable(Table table, List<String>
processorCapabilitie
throw new MetaException("Processor has no capabilities, cannot
create an ACID table.");
}
- newTable = validateTablePaths(table);
- if (MetaStoreUtils.isInsertOnlyTableParam(table.getParameters())) { //
MICRO_MANAGED Tables
+ validateTablePaths(newTable);
+ if (MetaStoreUtils.isInsertOnlyTableParam(newTable.getParameters())) {
// MICRO_MANAGED Tables
if (processorCapabilities.contains(HIVEMANAGEDINSERTWRITE)) {
LOG.debug("Processor has required capabilities to be able to
create INSERT-only tables");
return newTable;
@@ -694,7 +704,8 @@ public Table transformCreateTable(Table table, List<String>
processorCapabilitie
}
} else if (TableType.EXTERNAL_TABLE.name().equals(tableType)) {
LOG.debug("Table to be created is of type " + tableType);
- newTable = validateTablePaths(table);
+ params.put(HiveMetaHook.EXTERNAL, "TRUE");
+ validateTablePaths(newTable);
}
LOG.info("Transformer returning table:" + newTable.toString());
return newTable;
@@ -734,7 +745,7 @@ public Table transformAlterTable(Table oldTable, Table
newTable, List<String> pr
LOG.info("Starting translation for Alter table for processor " +
processorId + " with " + processorCapabilities
+ " on table " + newTable.getTableName());
-
+ validateIfAcidTablePermitted(newTable);
if (tableLocationChanged(oldTable, newTable)) {
validateTablePaths(newTable);
}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/leader/CompactorTasks.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/leader/CompactorTasks.java
index 956a5e5371f..efc9673c5ab 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/leader/CompactorTasks.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/leader/CompactorTasks.java
@@ -65,8 +65,13 @@ public List<MetaStoreThread> getCompactorThreads() throws
Exception {
compactors.add(initiator);
}
if (MetastoreConf.getBoolVar(configuration,
MetastoreConf.ConfVars.COMPACTOR_CLEANER_ON)) {
- MetaStoreThread cleaner =
instantiateThread("org.apache.hadoop.hive.ql.txn.compactor.Cleaner");
- compactors.add(cleaner);
+ if (MetastoreConf.getBoolVar(configuration,
MetastoreConf.ConfVars.METASTORE_SUPPORT_ACID)) {
+ MetaStoreThread cleaner =
instantiateThread("org.apache.hadoop.hive.ql.txn.compactor.Cleaner");
+ compactors.add(cleaner);
+ } else {
+ HiveMetaStore.LOG.warn("Compactor Cleaner is turned On. But, automatic
compaction cleaner will not run " +
+ "when the {} property is set to false.",
MetastoreConf.ConfVars.METASTORE_SUPPORT_ACID.getHiveName());
+ }
}
return compactors;
}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/leader/HouseKeepingTasks.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/leader/HouseKeepingTasks.java
index af3bd2b0ac0..0306d47313b 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/leader/HouseKeepingTasks.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/leader/HouseKeepingTasks.java
@@ -20,7 +20,6 @@
import com.cronutils.utils.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.metastore.MetastoreTaskThread;
@@ -107,20 +106,19 @@ public void takeLeadership(LeaderElection election)
throws Exception {
} else {
tasks = new ArrayList<>(getRemoteOnlyTasks());
}
- int poolSize = Math.min(MetastoreConf.getIntVar(configuration,
- MetastoreConf.ConfVars.THREAD_POOL_SIZE), tasks.size());
- metastoreTaskThreadPool = Executors.newScheduledThreadPool(poolSize,
threadFactory);
- for (MetastoreTaskThread task : tasks) {
+ tasks.forEach(task -> {
task.setConf(configuration);
task.enforceMutex(election.enforceMutex());
- long freq = task.runFrequency(TimeUnit.MILLISECONDS);
- if (freq > 0) {
+ if (task.runFrequency(TimeUnit.MILLISECONDS) > 0) {
runningTasks.add(task);
- metastoreTaskThreadPool.scheduleAtFixedRate(task, freq, freq,
TimeUnit.MILLISECONDS);
}
- }
-
+ });
+ int poolSize = Math.min(MetastoreConf.getIntVar(configuration,
+ MetastoreConf.ConfVars.THREAD_POOL_SIZE), runningTasks.size());
+ metastoreTaskThreadPool = Executors.newScheduledThreadPool(poolSize,
threadFactory);
runningTasks.forEach(task -> {
+ long freq = task.runFrequency(TimeUnit.MILLISECONDS);
+ metastoreTaskThreadPool.scheduleAtFixedRate(task, freq, freq,
TimeUnit.MILLISECONDS);
HiveMetaStore.LOG.info("Scheduling for " +
task.getClass().getCanonicalName() + " service.");
});
}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricLogger.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricLogger.java
index 4fbfced8328..24fc17a2a22 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricLogger.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricLogger.java
@@ -50,8 +50,8 @@ public class AcidMetricLogger implements MetastoreTaskThread {
@Override
public long runFrequency(TimeUnit timeUnit) {
- return MetastoreConf
- .getTimeVar(conf,
MetastoreConf.ConfVars.COMPACTOR_ACID_METRICS_LOGGER_FREQUENCY, timeUnit);
+ return MetastoreConf.getBoolVar(conf,
MetastoreConf.ConfVars.METASTORE_SUPPORT_ACID) ?
+ MetastoreConf.getTimeVar(conf,
MetastoreConf.ConfVars.COMPACTOR_ACID_METRICS_LOGGER_FREQUENCY, timeUnit) : 0;
}
@Override
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricService.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricService.java
index d80f84219ee..23200e50235 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricService.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/AcidMetricService.java
@@ -103,7 +103,8 @@ public class AcidMetricService implements
MetastoreTaskThread {
@Override
public long runFrequency(TimeUnit unit) {
- return MetastoreConf.getTimeVar(conf,
MetastoreConf.ConfVars.METASTORE_ACIDMETRICS_CHECK_INTERVAL, unit);
+ return MetastoreConf.getBoolVar(conf,
MetastoreConf.ConfVars.METASTORE_SUPPORT_ACID) ?
+ MetastoreConf.getTimeVar(conf,
MetastoreConf.ConfVars.METASTORE_ACIDMETRICS_CHECK_INTERVAL, unit) : 0;
}
@Override
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/service/AcidHouseKeeperService.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/service/AcidHouseKeeperService.java
index 836b85851e7..84a92e40367 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/service/AcidHouseKeeperService.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/service/AcidHouseKeeperService.java
@@ -75,7 +75,8 @@ public Configuration getConf() {
@Override
public long runFrequency(TimeUnit unit) {
- return MetastoreConf.getTimeVar(conf,
MetastoreConf.ConfVars.ACID_HOUSEKEEPER_SERVICE_INTERVAL, unit);
+ return MetastoreConf.getBoolVar(conf,
MetastoreConf.ConfVars.METASTORE_SUPPORT_ACID) ?
+ MetastoreConf.getTimeVar(conf,
MetastoreConf.ConfVars.ACID_HOUSEKEEPER_SERVICE_INTERVAL, unit) : 0;
}
@Override
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/service/AcidOpenTxnsCounterService.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/service/AcidOpenTxnsCounterService.java
index 89dbff3f96f..be39e615cb6 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/service/AcidOpenTxnsCounterService.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/service/AcidOpenTxnsCounterService.java
@@ -42,7 +42,8 @@ public class AcidOpenTxnsCounterService implements
MetastoreTaskThread {
@Override
public long runFrequency(TimeUnit unit) {
- return MetastoreConf.getTimeVar(conf,
MetastoreConf.ConfVars.COUNT_OPEN_TXNS_INTERVAL, unit);
+ return MetastoreConf.getBoolVar(conf,
MetastoreConf.ConfVars.METASTORE_SUPPORT_ACID) ?
+ MetastoreConf.getTimeVar(conf,
MetastoreConf.ConfVars.COUNT_OPEN_TXNS_INTERVAL, unit) : 0;
}
@Override
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/service/AcidTxnCleanerService.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/service/AcidTxnCleanerService.java
index 766ef7b67d8..227d846ae27 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/service/AcidTxnCleanerService.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/service/AcidTxnCleanerService.java
@@ -53,7 +53,8 @@ public Configuration getConf() {
@Override
public long runFrequency(TimeUnit unit) {
- return MetastoreConf.getTimeVar(conf,
MetastoreConf.ConfVars.ACID_TXN_CLEANER_INTERVAL, unit);
+ return MetastoreConf.getBoolVar(conf,
MetastoreConf.ConfVars.METASTORE_SUPPORT_ACID) ?
+ MetastoreConf.getTimeVar(conf,
MetastoreConf.ConfVars.ACID_TXN_CLEANER_INTERVAL, unit) : 0;
}
@Override
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/service/CompactionHouseKeeperService.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/service/CompactionHouseKeeperService.java
index 6eca4828344..653ccc50d6e 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/service/CompactionHouseKeeperService.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/service/CompactionHouseKeeperService.java
@@ -36,11 +36,14 @@ public CompactionHouseKeeperService() {
@Override
protected void initTasks(){
- tasks = ImmutableMap.<FailableRunnable<MetaException>, String>builder()
- .put(txnHandler::removeDuplicateCompletedTxnComponents,
- "Cleaning duplicate COMPLETED_TXN_COMPONENTS entries")
- .put(txnHandler::purgeCompactionHistory, "Cleaning obsolete
compaction history entries")
- .build();
+ ImmutableMap.Builder<FailableRunnable<MetaException>, String> taskBuilder =
+ ImmutableMap.<FailableRunnable<MetaException>, String>builder()
+ .put(txnHandler::purgeCompactionHistory, "Cleaning obsolete
compaction history entries");
+ if (MetastoreConf.getBoolVar(getConf(),
MetastoreConf.ConfVars.METASTORE_SUPPORT_ACID)) {
+ taskBuilder.put(txnHandler::removeDuplicateCompletedTxnComponents,
+ "Cleaning duplicate COMPLETED_TXN_COMPONENTS entries");
+ }
+ tasks = taskBuilder.build();
}
@Override
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestNoAcidSupport.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestNoAcidSupport.java
new file mode 100644
index 00000000000..95c0ca7a57f
--- /dev/null
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestNoAcidSupport.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.thrift.TException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Tests to verify metastore without acid support.
+ */
+@Category(MetastoreUnitTest.class)
+public class TestNoAcidSupport {
+ private static Configuration conf;
+ private static HiveMetaStoreClient client;
+ private static final String DB_NAME = "TestNoAcidSupport";
+ private static final String TABLE_NAME = "t";
+
+ @BeforeClass
+ public static void beforeTests() throws Exception {
+ conf = MetastoreConf.newMetastoreConf();
+ MetastoreConf.setBoolVar(conf, ConfVars.METASTORE_SUPPORT_ACID, false);
+ client = new HiveMetaStoreClient(conf);
+ client.dropDatabase(DB_NAME, true, true, true);
+ new DatabaseBuilder().setName(DB_NAME).create(client, conf);
+ }
+
+ @AfterClass
+ public static void afterTests() throws Exception {
+ try {
+ client.dropDatabase(DB_NAME, true, true, true);
+ client.close();
+ } catch (Throwable e) {
+ System.err.println(StringUtils.stringifyException(e));
+ throw e;
+ }
+ }
+
+ @After
+ public void afterTest() throws TException {
+ client.dropTable(DB_NAME, TABLE_NAME);
+ }
+
+ @Test
+ public void testCreateManagedAcidTable() {
+ Exception exception = Assert.assertThrows(MetaException.class, () -> {
+ new TableBuilder().setDbName(DB_NAME).setTableName(TABLE_NAME)
+ .addCol("i", ColumnType.INT_TYPE_NAME)
+ .setType(TableType.MANAGED_TABLE.name())
+ .addTableParam(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL,
"true")
+ .create(client, conf);
+ });
+ Assert.assertTrue(exception.getMessage().contains("ACID tables are not
permitted when the " +
+ ConfVars.METASTORE_SUPPORT_ACID.getHiveName() + " property is set to
false"));
+ }
+
+ @Test
+ public void testCreateManagedTranslateToExternalTable() throws Exception {
+ new TableBuilder().setDbName(DB_NAME).setTableName(TABLE_NAME)
+ .addCol("i", ColumnType.INT_TYPE_NAME)
+ .setType(TableType.MANAGED_TABLE.name())
+ .create(client, conf);
+ Table t = client.getTable(DB_NAME, TABLE_NAME);
+ Assert.assertEquals(TableType.EXTERNAL_TABLE.name(), t.getTableType());
+
Assert.assertTrue(Boolean.parseBoolean(t.getParameters().get(HiveMetaHook.EXTERNAL)));
+
Assert.assertTrue(Boolean.parseBoolean(t.getParameters().get(MetaStoreUtils.EXTERNAL_TABLE_PURGE)));
+ }
+
+ @Test
+ public void testCreateExternalTable() throws Exception {
+ new TableBuilder().setDbName(DB_NAME).setTableName(TABLE_NAME)
+ .addCol("i", ColumnType.INT_TYPE_NAME)
+ .setType(TableType.EXTERNAL_TABLE.name())
+ .create(client, conf);
+ Table t = client.getTable(DB_NAME, TABLE_NAME);
+ Assert.assertEquals(TableType.EXTERNAL_TABLE.name(), t.getTableType());
+
Assert.assertTrue(Boolean.parseBoolean(t.getParameters().get(HiveMetaHook.EXTERNAL)));
+
Assert.assertNull(t.getParameters().get(MetaStoreUtils.EXTERNAL_TABLE_PURGE));
+ }
+
+ @Test
+ public void testAlterToManagedAcidTable() throws Exception {
+ new TableBuilder().setDbName(DB_NAME).setTableName(TABLE_NAME)
+ .addCol("i", ColumnType.INT_TYPE_NAME)
+ .setType(TableType.EXTERNAL_TABLE.name())
+ .create(client, conf);
+ Table t = client.getTable(DB_NAME, TABLE_NAME);
+ Assert.assertEquals(TableType.EXTERNAL_TABLE.name(), t.getTableType());
+ t.setTableType(TableType.MANAGED_TABLE.name());
+ t.getParameters().put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL,
"true");
+ Exception exception = Assert.assertThrows(MetaException.class, () -> {
+ client.alter_table(DB_NAME, TABLE_NAME, t);
+ });
+ Assert.assertTrue(exception.getMessage().contains("ACID tables are not
permitted when the " +
+ ConfVars.METASTORE_SUPPORT_ACID.getHiveName() + " property is set to
false"));
+ }
+}
