Repository: hive Updated Branches: refs/heads/master 0772f186f -> faf29143d
HIVE-20196 : Remove MetastoreConf dependency on server-specific classes (Alexander Kolbasov reviewed by Alan Gates and Vihang Karajgaonkar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/faf29143 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/faf29143 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/faf29143 Branch: refs/heads/master Commit: faf29143dd94effc6a7192f8a04a79d38d6382f7 Parents: 0772f18 Author: Alexander Kolbasov <[email protected]> Authored: Tue Aug 14 10:48:36 2018 -0700 Committer: Vihang Karajgaonkar <[email protected]> Committed: Tue Aug 14 10:48:36 2018 -0700 ---------------------------------------------------------------------- .../hive/metastore/conf/MetastoreConf.java | 67 +++++++++++++------- .../hive/metastore/conf/TestMetastoreConf.java | 44 +++++++++++++ 2 files changed, 89 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/faf29143/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index cf5fbbe..30ea7f8 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -19,17 +19,6 @@ package org.apache.hadoop.hive.metastore.conf; import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader; -import org.apache.hadoop.hive.metastore.HiveAlterHandler; -import org.apache.hadoop.hive.metastore.MaterializationsRebuildLockCleanerTask; -import org.apache.hadoop.hive.metastore.MetastoreTaskThread; -import org.apache.hadoop.hive.metastore.RuntimeStatsCleanerTask; -import org.apache.hadoop.hive.metastore.events.EventCleanerTask; -import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager; -import org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService; -import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService; -import org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService; -import org.apache.hadoop.hive.metastore.txn.AcidWriteSetService; import org.apache.hadoop.hive.metastore.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,6 +59,40 @@ public class MetastoreConf { private static final Logger LOG = LoggerFactory.getLogger(MetastoreConf.class); private static final Pattern TIME_UNIT_SUFFIX = Pattern.compile("([0-9]+)([a-zA-Z]+)"); + @VisibleForTesting + static final String DEFAULT_STORAGE_SCHEMA_READER_CLASS = + "org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader"; + @VisibleForTesting + static final String HIVE_ALTER_HANDLE_CLASS = + "org.apache.hadoop.hive.metastore.HiveAlterHandler"; + @VisibleForTesting + static final String MATERIALZIATIONS_REBUILD_LOCK_CLEANER_TASK_CLASS = + "org.apache.hadoop.hive.metastore.MaterializationsRebuildLockCleanerTask"; + @VisibleForTesting + static final String METASTORE_TASK_THREAD_CLASS = + "org.apache.hadoop.hive.metastore.MetastoreTaskThread"; + @VisibleForTesting + static final String RUNTIME_STATS_CLEANER_TASK_CLASS = + "org.apache.hadoop.hive.metastore.RuntimeStatsCleanerTask"; + @VisibleForTesting + static final String EVENT_CLEANER_TASK_CLASS = + "org.apache.hadoop.hive.metastore.events.EventCleanerTask"; + @VisibleForTesting + static final String METASTORE_DELEGATION_MANAGER_CLASS = + "org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager"; + @VisibleForTesting + static final String ACID_COMPACTION_HISTORY_SERVICE_CLASS = + "org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService"; + @VisibleForTesting + static final String ACID_HOUSE_KEEPER_SERVICE_CLASS = + "org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService"; + @VisibleForTesting + static final String ACID_OPEN_TXNS_COUNTER_SERVICE_CLASS = + "org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService"; + @VisibleForTesting + static final String ACID_WRITE_SET_SERVICE_CLASS = + "org.apache.hadoop.hive.metastore.txn.AcidWriteSetService"; + private static final Map<String, ConfVars> metaConfs = new HashMap<>(); private static URL hiveDefaultURL = null; private static URL hiveSiteURL = null; @@ -260,7 +283,7 @@ public class MetastoreConf { "hive.metastore.aggregate.stats.cache.ttl", 600, TimeUnit.SECONDS, "Number of seconds for a cached node to be active in the cache before they become stale."), ALTER_HANDLER("metastore.alter.handler", "hive.metastore.alter.impl", - HiveAlterHandler.class.getName(), + HIVE_ALTER_HANDLE_CLASS, "Alter handler. For now defaults to the Hive one. Really need a better default option"), ASYNC_LOG_ENABLED("metastore.async.log.enabled", "hive.async.log.enabled", true, "Whether to enable Log4j2's asynchronous logging. Asynchronous logging can give\n" + @@ -436,7 +459,7 @@ public class MetastoreConf { DELEGATION_TOKEN_RENEW_INTERVAL("metastore.cluster.delegation.token.renew-interval", "hive.cluster.delegation.token.renew-interval", 1, TimeUnit.DAYS, ""), DELEGATION_TOKEN_STORE_CLS("metastore.cluster.delegation.token.store.class", - "hive.cluster.delegation.token.store.class", MetastoreDelegationTokenManager.class.getName(), + "hive.cluster.delegation.token.store.class", METASTORE_DELEGATION_MANAGER_CLASS, "Class to store delegation tokens"), DETACH_ALL_ON_COMMIT("javax.jdo.option.DetachAllOnCommit", "javax.jdo.option.DetachAllOnCommit", true, @@ -747,7 +770,7 @@ public class MetastoreConf { "hive.metastore.stats.auto.analyze.worker.count", 1, "Number of parallel analyze commands to run for background stats update."), STORAGE_SCHEMA_READER_IMPL("metastore.storage.schema.reader.impl", "metastore.storage.schema.reader.impl", - DefaultStorageSchemaReader.class.getName(), + DEFAULT_STORAGE_SCHEMA_READER_CLASS, "The class to use to read schemas from storage. It must implement " + "org.apache.hadoop.hive.metastore.StorageSchemaReader"), STORE_MANAGER_TYPE("datanucleus.storeManagerType", "datanucleus.storeManagerType", "rdbms", "metadata store type"), @@ -761,21 +784,21 @@ public class MetastoreConf { + "The only supported special character right now is '/'. This flag applies only to quoted table names.\n" + "The default value is true."), TASK_THREADS_ALWAYS("metastore.task.threads.always", "metastore.task.threads.always", - EventCleanerTask.class.getName() + "," + RuntimeStatsCleanerTask.class.getName() + "," + + EVENT_CLEANER_TASK_CLASS + "," + RUNTIME_STATS_CLEANER_TASK_CLASS + "," + "org.apache.hadoop.hive.metastore.repl.DumpDirCleanerTask" + "," + "org.apache.hadoop.hive.metastore.HiveProtoEventsCleanerTask", "Comma separated list of tasks that will be started in separate threads. These will " + "always be started, regardless of whether the metastore is running in embedded mode " + - "or in server mode. They must implement " + MetastoreTaskThread.class.getName()), + "or in server mode. They must implement " + METASTORE_TASK_THREAD_CLASS), TASK_THREADS_REMOTE_ONLY("metastore.task.threads.remote", "metastore.task.threads.remote", - AcidHouseKeeperService.class.getName() + "," + - AcidOpenTxnsCounterService.class.getName() + "," + - AcidCompactionHistoryService.class.getName() + "," + - AcidWriteSetService.class.getName() + "," + - MaterializationsRebuildLockCleanerTask.class.getName(), + ACID_HOUSE_KEEPER_SERVICE_CLASS + "," + + ACID_OPEN_TXNS_COUNTER_SERVICE_CLASS + "," + + ACID_COMPACTION_HISTORY_SERVICE_CLASS + "," + + ACID_WRITE_SET_SERVICE_CLASS + "," + + MATERIALZIATIONS_REBUILD_LOCK_CLEANER_TASK_CLASS, "Command separated list of tasks that will be started in separate threads. These will be" + " started only when the metastore is running as a separate service. They must " + - "implement " + MetastoreTaskThread.class.getName()), + "implement " + METASTORE_TASK_THREAD_CLASS), TCP_KEEP_ALIVE("metastore.server.tcp.keepalive", "hive.metastore.server.tcp.keepalive", true, "Whether to enable TCP keepalive for the metastore server. Keepalive will prevent accumulation of half-open connections."), http://git-wip-us.apache.org/repos/asf/hive/blob/faf29143/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java index 1422701..9905a14 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java @@ -40,6 +40,18 @@ import java.util.Map; import java.util.Random; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader; +import org.apache.hadoop.hive.metastore.HiveAlterHandler; +import org.apache.hadoop.hive.metastore.MaterializationsRebuildLockCleanerTask; +import org.apache.hadoop.hive.metastore.MetastoreTaskThread; +import org.apache.hadoop.hive.metastore.RuntimeStatsCleanerTask; +import org.apache.hadoop.hive.metastore.events.EventCleanerTask; +import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager; +import org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService; +import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService; +import org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService; +import org.apache.hadoop.hive.metastore.txn.AcidWriteSetService; + @Category(MetastoreUnitTest.class) public class TestMetastoreConf { @@ -430,4 +442,36 @@ public class TestMetastoreConf { // Make sure the hidden keys didn't get published Assert.assertThat(dump, CoreMatchers.not(new StringContains(ConfVars.PWD.getVarname()))); } + + /** + * Test class names hardcoded in MetastoreConf. + * MetastoreConf uses several hard-coded class names. If one of these classes is renamed or + * moved to a different package we want to be able to catch this. So we compare expected + * class name with the actual one. + */ + @Test + public void testClassNames() { + Assert.assertEquals(MetastoreConf.DEFAULT_STORAGE_SCHEMA_READER_CLASS, + DefaultStorageSchemaReader.class.getName()); + Assert.assertEquals(MetastoreConf.HIVE_ALTER_HANDLE_CLASS, + HiveAlterHandler.class.getName()); + Assert.assertEquals(MetastoreConf.MATERIALZIATIONS_REBUILD_LOCK_CLEANER_TASK_CLASS, + MaterializationsRebuildLockCleanerTask.class.getName()); + Assert.assertEquals(MetastoreConf.METASTORE_TASK_THREAD_CLASS, + MetastoreTaskThread.class.getName()); + Assert.assertEquals(MetastoreConf.RUNTIME_STATS_CLEANER_TASK_CLASS, + RuntimeStatsCleanerTask.class.getName()); + Assert.assertEquals(MetastoreConf.EVENT_CLEANER_TASK_CLASS, + EventCleanerTask.class.getName()); + Assert.assertEquals(MetastoreConf.METASTORE_DELEGATION_MANAGER_CLASS, + MetastoreDelegationTokenManager.class.getName()); + Assert.assertEquals(MetastoreConf.ACID_COMPACTION_HISTORY_SERVICE_CLASS, + AcidCompactionHistoryService.class.getName()); + Assert.assertEquals(MetastoreConf.ACID_HOUSE_KEEPER_SERVICE_CLASS, + AcidHouseKeeperService.class.getName()); + Assert.assertEquals(MetastoreConf.ACID_OPEN_TXNS_COUNTER_SERVICE_CLASS, + AcidOpenTxnsCounterService.class.getName()); + Assert.assertEquals(MetastoreConf.ACID_WRITE_SET_SERVICE_CLASS, + AcidWriteSetService.class.getName()); + } }
