Repository: hadoop Updated Branches: refs/heads/branch-2.9 77c6439c3 -> 050aa531b
HDFS-13099. RBF: Use the ZooKeeper as the default State Store. Contributed by Yiqun Lin. (cherry picked from commit 543f3abbee79d7ec70353f0cdda6397ee001324e) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/050aa531 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/050aa531 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/050aa531 Branch: refs/heads/branch-2.9 Commit: 050aa531ba7a419f83a4f48b0dac1ffbbdad6d1b Parents: 77c6439 Author: Yiqun Lin <yq...@apache.org> Authored: Fri Feb 9 13:57:42 2018 +0800 Committer: Yiqun Lin <yq...@apache.org> Committed: Fri Feb 9 14:03:13 2018 +0800 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++-- .../src/main/resources/hdfs-default.xml | 10 ++++++++-- .../src/site/markdown/HDFSRouterFederation.md | 2 +- .../server/federation/RouterConfigBuilder.java | 6 ++++++ .../store/FederationStateStoreTestUtils.java | 17 +++++++++++------ 5 files changed, 28 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/050aa531/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index e1d5806..dac9e71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -33,8 +33,8 @@ import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCPerformance import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver; import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver; import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver; -import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl; import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl; +import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl; import org.apache.hadoop.http.HttpConfig; /** @@ -1115,7 +1115,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String FEDERATION_STORE_DRIVER_CLASS = FEDERATION_STORE_PREFIX + "driver.class"; public static final Class<? extends StateStoreDriver> - FEDERATION_STORE_DRIVER_CLASS_DEFAULT = StateStoreFileImpl.class; + FEDERATION_STORE_DRIVER_CLASS_DEFAULT = StateStoreZooKeeperImpl.class; public static final String FEDERATION_STORE_CONNECTION_TEST_MS = FEDERATION_STORE_PREFIX + "connection.test"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/050aa531/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index cb4616e..5bbd853 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -4614,9 +4614,15 @@ <property> <name>dfs.federation.router.store.driver.class</name> - <value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl</value> + <value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl</value> <description> - Class to implement the State Store. By default it uses the local disk. + Class to implement the State Store. There are three implementation classes currently + being supported: + org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl, + org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl and + org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl. + These implementation classes use the local file, filesystem and ZooKeeper as a backend respectively. + By default it uses the ZooKeeper as the default State Store. </description> </property> http://git-wip-us.apache.org/repos/asf/hadoop/blob/050aa531/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md index b1e9ac2..713fa7a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md @@ -305,7 +305,7 @@ The connection to the State Store and the internal caching at the Router. |:---- |:---- |:---- | | dfs.federation.router.store.enable | `true` | If `true`, the Router connects to the State Store. | | dfs.federation.router.store.serializer | `StateStoreSerializerPBImpl` | Class to serialize State Store records. | -| dfs.federation.router.store.driver.class | `StateStoreZKImpl` | Class to implement the State Store. | +| dfs.federation.router.store.driver.class | `StateStoreZooKeeperImpl` | Class to implement the State Store. | | dfs.federation.router.store.connection.test | 60000 | How often to check for the connection to the State Store in milliseconds. | | dfs.federation.router.cache.ttl | 60000 | How often to refresh the State Store caches in milliseconds. | | dfs.federation.router.store.membership.expiration | 300000 | Expiration time in milliseconds for a membership record. | http://git-wip-us.apache.org/repos/asf/hadoop/blob/050aa531/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java index 0d908af..aa7eca2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hdfs.server.federation; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils; +import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver; /** * Constructs a router configuration with individual features enabled/disabled. @@ -113,6 +115,10 @@ public class RouterConfigBuilder { } public RouterConfigBuilder stateStore() { + // reset the State Store driver implementation class for testing + conf.setClass(DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS, + FederationStateStoreTestUtils.getTestDriverClass(), + StateStoreDriver.class); return this.stateStore(true); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/050aa531/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java index dbb8f3f..def3935 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState; import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver; import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileBaseImpl; +import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl; import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord; import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState; import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats; @@ -50,17 +51,21 @@ import org.apache.hadoop.util.Time; */ public final class FederationStateStoreTestUtils { + /** The State Store Driver implementation class for testing .*/ + private static final Class<? extends StateStoreDriver> + FEDERATION_STORE_DRIVER_CLASS_FOR_TEST = StateStoreFileImpl.class; + private FederationStateStoreTestUtils() { // Utility Class } /** - * Get the default State Store driver implementation. + * Get the State Store driver implementation for testing. * - * @return Class of the default State Store driver implementation. + * @return Class of the State Store driver implementation. */ - public static Class<? extends StateStoreDriver> getDefaultDriver() { - return DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS_DEFAULT; + public static Class<? extends StateStoreDriver> getTestDriverClass() { + return FEDERATION_STORE_DRIVER_CLASS_FOR_TEST; } /** @@ -69,7 +74,7 @@ public final class FederationStateStoreTestUtils { * @return State Store configuration. */ public static Configuration getStateStoreConfiguration() { - Class<? extends StateStoreDriver> clazz = getDefaultDriver(); + Class<? extends StateStoreDriver> clazz = getTestDriverClass(); return getStateStoreConfiguration(clazz); } @@ -146,7 +151,7 @@ public final class FederationStateStoreTestUtils { * @throws IOException */ public static void deleteStateStore() throws IOException { - Class<? extends StateStoreDriver> driverClass = getDefaultDriver(); + Class<? extends StateStoreDriver> driverClass = getTestDriverClass(); deleteStateStore(driverClass); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
