[hadoop] 26/50: HDFS-13523. Support observer nodes in MiniDFSCluster. Contributed by Konstantin Shvachko.
This is an automated email from the ASF dual-hosted git repository.
cliang pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 7b425c87030b3e720e1ebc69f3fdbc8d867654b6
Author: Konstantin V Shvachko
AuthorDate: Thu Oct 11 22:39:17 2018 -0700
HDFS-13523. Support observer nodes in MiniDFSCluster. Contributed by
Konstantin Shvachko.
---
.../hdfs/TestStateAlignmentContextWithHA.java | 61 ++
.../hadoop/hdfs/server/namenode/ha/HATestUtil.java | 40 ++
.../namenode/ha/TestConsistentReadsObserver.java | 19 ++-
.../server/namenode/ha/TestMultiObserverNode.java | 16 ++
.../hdfs/server/namenode/ha/TestObserverNode.java | 16 ++
5 files changed, 67 insertions(+), 85 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStateAlignmentContextWithHA.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStateAlignmentContextWithHA.java
index a642872..3dbeea7 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStateAlignmentContextWithHA.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStateAlignmentContextWithHA.java
@@ -18,14 +18,15 @@
package org.apache.hadoop.hdfs;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster;
import org.apache.hadoop.hdfs.server.namenode.ha.HAProxyFactory;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider;
@@ -66,6 +67,7 @@ public class TestStateAlignmentContextWithHA {
private static final Configuration CONF = new HdfsConfiguration();
private static final List AC_LIST = new ArrayList<>();
+ private static MiniQJMHACluster qjmhaCluster;
private static MiniDFSCluster cluster;
private static List clients;
@@ -87,33 +89,26 @@ public class TestStateAlignmentContextWithHA {
@BeforeClass
public static void startUpCluster() throws IOException {
-// disable block scanner
-CONF.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1);
// Set short retry timeouts so this test runs faster
CONF.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10);
-CONF.setBoolean("fs.hdfs.impl.disable.cache", true);
+CONF.setBoolean(String.format(
+"fs.%s.impl.disable.cache", HdfsConstants.HDFS_URI_SCHEME), true);
+CONF.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, NUMDATANODES);
-cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(NUMDATANODES)
-.nnTopology(MiniDFSNNTopology.simpleHATopology(3))
-.build();
-cluster.waitActive();
-cluster.transitionToActive(0);
-cluster.transitionToObserver(2);
-
-HATestUtil.setupHAConfiguration(
-cluster, CONF, 0, ORPPwithAlignmentContexts.class);
+qjmhaCluster = HATestUtil.setUpObserverCluster(CONF, 1, NUMDATANODES,
true);
+cluster = qjmhaCluster.getDfsCluster();
}
@Before
public void before() throws IOException, URISyntaxException {
-dfs = (DistributedFileSystem) FileSystem.get(CONF);
+dfs = HATestUtil.configureObserverReadFs(
+cluster, CONF, ORPPwithAlignmentContexts.class, true);
}
@AfterClass
public static void shutDownCluster() throws IOException {
-if (cluster != null) {
- cluster.shutdown();
- cluster = null;
+if (qjmhaCluster != null) {
+ qjmhaCluster.shutdown();
}
}
@@ -144,9 +139,9 @@ public class TestStateAlignmentContextWithHA {
long postWriteState =
cluster.getNamesystem(active).getLastWrittenTransactionId();
// Write(s) should have increased state. Check for greater than.
-assertThat(clientState > preWriteState, is(true));
+assertTrue(clientState > preWriteState);
// Client and server state should be equal.
-assertThat(clientState, is(postWriteState));
+assertEquals(clientState, postWriteState);
}
/**
@@ -161,7 +156,7 @@ public class TestStateAlignmentContextWithHA {
DFSTestUtil.readFile(dfs, new Path("/testFile2"));
// Read should catch client up to last written state.
long clientState = getContext(0).getLastSeenStateId();
-assertThat(clientState, is(lastWrittenId));
+assertEquals(clientState, lastWrittenId);
}
/**
@@ -173,12 +168,12 @@ public class TestStateAlignmentContextWithHA {
DFSTestUtil.writeFile(dfs, new Path("/testFile3"), "ezpz");
long lastWrittenId =
[hadoop] 26/50: HDFS-13523. Support observer nodes in MiniDFSCluster. Contributed by Konstantin Shvachko.
This is an automated email from the ASF dual-hosted git repository.
cliang pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit dea7e1a239e1133eab659083c88446a076854915
Author: Konstantin V Shvachko
AuthorDate: Thu Oct 11 22:39:17 2018 -0700
HDFS-13523. Support observer nodes in MiniDFSCluster. Contributed by
Konstantin Shvachko.
---
.../hdfs/TestStateAlignmentContextWithHA.java | 61 ++
.../hadoop/hdfs/server/namenode/ha/HATestUtil.java | 40 ++
.../namenode/ha/TestConsistentReadsObserver.java | 19 ++-
.../server/namenode/ha/TestMultiObserverNode.java | 16 ++
.../hdfs/server/namenode/ha/TestObserverNode.java | 16 ++
5 files changed, 67 insertions(+), 85 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStateAlignmentContextWithHA.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStateAlignmentContextWithHA.java
index a642872..3dbeea7 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStateAlignmentContextWithHA.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStateAlignmentContextWithHA.java
@@ -18,14 +18,15 @@
package org.apache.hadoop.hdfs;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster;
import org.apache.hadoop.hdfs.server.namenode.ha.HAProxyFactory;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider;
@@ -66,6 +67,7 @@ public class TestStateAlignmentContextWithHA {
private static final Configuration CONF = new HdfsConfiguration();
private static final List AC_LIST = new ArrayList<>();
+ private static MiniQJMHACluster qjmhaCluster;
private static MiniDFSCluster cluster;
private static List clients;
@@ -87,33 +89,26 @@ public class TestStateAlignmentContextWithHA {
@BeforeClass
public static void startUpCluster() throws IOException {
-// disable block scanner
-CONF.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1);
// Set short retry timeouts so this test runs faster
CONF.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10);
-CONF.setBoolean("fs.hdfs.impl.disable.cache", true);
+CONF.setBoolean(String.format(
+"fs.%s.impl.disable.cache", HdfsConstants.HDFS_URI_SCHEME), true);
+CONF.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, NUMDATANODES);
-cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(NUMDATANODES)
-.nnTopology(MiniDFSNNTopology.simpleHATopology(3))
-.build();
-cluster.waitActive();
-cluster.transitionToActive(0);
-cluster.transitionToObserver(2);
-
-HATestUtil.setupHAConfiguration(
-cluster, CONF, 0, ORPPwithAlignmentContexts.class);
+qjmhaCluster = HATestUtil.setUpObserverCluster(CONF, 1, NUMDATANODES,
true);
+cluster = qjmhaCluster.getDfsCluster();
}
@Before
public void before() throws IOException, URISyntaxException {
-dfs = (DistributedFileSystem) FileSystem.get(CONF);
+dfs = HATestUtil.configureObserverReadFs(
+cluster, CONF, ORPPwithAlignmentContexts.class, true);
}
@AfterClass
public static void shutDownCluster() throws IOException {
-if (cluster != null) {
- cluster.shutdown();
- cluster = null;
+if (qjmhaCluster != null) {
+ qjmhaCluster.shutdown();
}
}
@@ -144,9 +139,9 @@ public class TestStateAlignmentContextWithHA {
long postWriteState =
cluster.getNamesystem(active).getLastWrittenTransactionId();
// Write(s) should have increased state. Check for greater than.
-assertThat(clientState > preWriteState, is(true));
+assertTrue(clientState > preWriteState);
// Client and server state should be equal.
-assertThat(clientState, is(postWriteState));
+assertEquals(clientState, postWriteState);
}
/**
@@ -161,7 +156,7 @@ public class TestStateAlignmentContextWithHA {
DFSTestUtil.readFile(dfs, new Path("/testFile2"));
// Read should catch client up to last written state.
long clientState = getContext(0).getLastSeenStateId();
-assertThat(clientState, is(lastWrittenId));
+assertEquals(clientState, lastWrittenId);
}
/**
@@ -173,12 +168,12 @@ public class TestStateAlignmentContextWithHA {
DFSTestUtil.writeFile(dfs, new Path("/testFile3"), "ezpz");
long lastWrittenId =
