hadoop git commit: HDFS-8209. Support different number of datanode directories in MiniDFSCluster. (Contributed by surendra singh lilhore)

2015-05-08 Thread vinayakumarb 
Repository: hadoop
Updated Branches:
  refs/heads/trunk ecfa05227 - 4c6816faf


HDFS-8209. Support different number of datanode directories in MiniDFSCluster. 
(Contributed by surendra singh lilhore)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c6816fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c6816fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c6816fa

Branch: refs/heads/trunk
Commit: 4c6816faf83726c71c9c3981e7280b461f824a4f
Parents: ecfa052
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri May 8 15:03:44 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri May 8 15:03:44 2015 +0530

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../org/apache/hadoop/hdfs/MiniDFSCluster.java  |  4 +-
 .../apache/hadoop/hdfs/TestMiniDFSCluster.java  | 43 
 3 files changed, 49 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c6816fa/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 195a946..82e0799 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -684,6 +684,9 @@ Release 2.8.0 - UNRELEASED
 HDFS-8175. Provide information on snapshotDiff for supporting the 
comparison
 between snapshot and current status (J.Andreina via vinayakumarb)
 
+HDFS-8209. Support different number of datanode directories in 
MiniDFSCluster.
+(surendra singh lilhore via vinayakumarb)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c6816fa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 8aeaef8..12ad23e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -1211,8 +1211,10 @@ public class MiniDFSCluster {
 
   String makeDataNodeDirs(int dnIndex, StorageType[] storageTypes) throws 
IOException {
 StringBuilder sb = new StringBuilder();
-assert storageTypes == null || storageTypes.length == storagesPerDatanode;
 for (int j = 0; j  storagesPerDatanode; ++j) {
+  if ((storageTypes != null)  (j = storageTypes.length)) {
+break;
+  }
   File dir = getInstanceStorageDir(dnIndex, j);
   dir.mkdirs();
   if (!dir.isDirectory()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c6816fa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
index cf29d97..3fa852e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
@@ -23,9 +23,13 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assume.assumeTrue;
 
 import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.test.PathUtils;
 import org.junit.Before;
 import org.junit.Test;
@@ -139,4 +143,43 @@ public class TestMiniDFSCluster {
   MiniDFSCluster.shutdownCluster(cluster5);
 }
   }
+
+  @Test
+  public void testClusterSetDatanodeDifferentStorageType() throws IOException {
+final Configuration conf = new HdfsConfiguration();
+StorageType[][] storageType = new StorageType[][] {
+{StorageType.DISK, StorageType.ARCHIVE}, {StorageType.DISK},
+{StorageType.ARCHIVE}};
+final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+.numDataNodes(3).storageTypes(storageType).build();
+try {
+  cluster.waitActive();
+  ArrayListDataNode dataNodes = cluster.getDataNodes();
+  // Check the number of directory in DN's
+  for (int i = 0; i  storageType.length; i++) {
+

[04/50] hadoop git commit: HDFS-8209. Support different number of datanode directories in MiniDFSCluster. (Contributed by surendra singh lilhore)

2015-05-08 Thread zjshen 
HDFS-8209. Support different number of datanode directories in MiniDFSCluster. 
(Contributed by surendra singh lilhore)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cd127aea
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cd127aea
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cd127aea

Branch: refs/heads/YARN-2928
Commit: cd127aeadf4fa84fe1b6a122e7fc2d0f8f9837fc
Parents: e0232ff
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri May 8 15:03:44 2015 +0530
Committer: Zhijie Shen zjs...@apache.org
Committed: Fri May 8 17:32:45 2015 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../org/apache/hadoop/hdfs/MiniDFSCluster.java  |  4 +-
 .../apache/hadoop/hdfs/TestMiniDFSCluster.java  | 43 
 3 files changed, 49 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd127aea/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 195a946..82e0799 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -684,6 +684,9 @@ Release 2.8.0 - UNRELEASED
 HDFS-8175. Provide information on snapshotDiff for supporting the 
comparison
 between snapshot and current status (J.Andreina via vinayakumarb)
 
+HDFS-8209. Support different number of datanode directories in 
MiniDFSCluster.
+(surendra singh lilhore via vinayakumarb)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd127aea/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 8aeaef8..12ad23e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -1211,8 +1211,10 @@ public class MiniDFSCluster {
 
   String makeDataNodeDirs(int dnIndex, StorageType[] storageTypes) throws 
IOException {
 StringBuilder sb = new StringBuilder();
-assert storageTypes == null || storageTypes.length == storagesPerDatanode;
 for (int j = 0; j  storagesPerDatanode; ++j) {
+  if ((storageTypes != null)  (j = storageTypes.length)) {
+break;
+  }
   File dir = getInstanceStorageDir(dnIndex, j);
   dir.mkdirs();
   if (!dir.isDirectory()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd127aea/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
index cf29d97..3fa852e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
@@ -23,9 +23,13 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assume.assumeTrue;
 
 import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.test.PathUtils;
 import org.junit.Before;
 import org.junit.Test;
@@ -139,4 +143,43 @@ public class TestMiniDFSCluster {
   MiniDFSCluster.shutdownCluster(cluster5);
 }
   }
+
+  @Test
+  public void testClusterSetDatanodeDifferentStorageType() throws IOException {
+final Configuration conf = new HdfsConfiguration();
+StorageType[][] storageType = new StorageType[][] {
+{StorageType.DISK, StorageType.ARCHIVE}, {StorageType.DISK},
+{StorageType.ARCHIVE}};
+final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+.numDataNodes(3).storageTypes(storageType).build();
+try {
+  cluster.waitActive();
+  ArrayListDataNode dataNodes = cluster.getDataNodes();
+  // Check the number of directory in DN's
+  for (int i = 0; i  storageType.length; i++) {
+assertEquals(DataNode.getStorageLocations(dataNodes.get(i).getConf())
+.size(),