[23/35] hadoop git commit: HDFS-13188. Disk Balancer: Support multiple block pools during block move. Contributed by Bharat Viswanadham.

2018-03-07 Thread aengineer 
HDFS-13188. Disk Balancer: Support multiple block pools during block move. 
Contributed by Bharat Viswanadham.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/70607256
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/70607256
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/70607256

Branch: refs/heads/HDFS-7240
Commit: 7060725662cb3317ff2f0fcc38f965fd23e8e6aa
Parents: e6f99e2
Author: Inigo Goiri 
Authored: Tue Mar 6 09:09:32 2018 -0800
Committer: Inigo Goiri 
Committed: Tue Mar 6 09:09:32 2018 -0800

--
 .../hdfs/server/datanode/DiskBalancer.java  |  4 +-
 .../diskbalancer/DiskBalancerTestUtil.java  | 21 --
 .../server/diskbalancer/TestDiskBalancer.java   | 72 +---
 .../diskbalancer/TestDiskBalancerRPC.java   |  4 +-
 4 files changed, 85 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/70607256/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
index e90a47e..91c3624 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
@@ -958,8 +958,8 @@ public class DiskBalancer {
   ExtendedBlock block = null;
   while (block == null && currentCount < poolIters.size()) {
 currentCount++;
-poolIndex = poolIndex++ % poolIters.size();
-FsVolumeSpi.BlockIterator currentPoolIter = poolIters.get(poolIndex);
+int index = poolIndex++ % poolIters.size();
+FsVolumeSpi.BlockIterator currentPoolIter = poolIters.get(index);
 block = getBlockToCopy(currentPoolIter, item);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/70607256/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java
index bc4181b..bd8dbce 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java
@@ -40,6 +40,10 @@ import 
org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolume;
 import 
org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolumeSet;
 import org.apache.hadoop.util.Time;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
@@ -51,6 +55,7 @@ import java.util.concurrent.TimeoutException;
  * Helper class to create various cluster configurations at run time.
  */
 public class DiskBalancerTestUtil {
+  static final Logger LOG = LoggerFactory.getLogger(TestDiskBalancer.class);
   public static final long MB = 1024 * 1024L;
   public static final long GB = MB * 1024L;
   public static final long TB = GB * 1024L;
@@ -241,17 +246,25 @@ public class DiskBalancerTestUtil {
* @return Number of Blocks.
* @throws IOException
*/
-  public static int getBlockCount(FsVolumeSpi source) throws IOException {
+  public static int getBlockCount(FsVolumeSpi source,
+  boolean checkblockPoolCount)
+  throws IOException {
 int count = 0;
 for (String blockPoolID : source.getBlockPoolList()) {
   FsVolumeSpi.BlockIterator sourceIter =
   source.newBlockIterator(blockPoolID, "TestDiskBalancerSource");
+  int blockCount = 0;
   while (!sourceIter.atEnd()) {
 ExtendedBlock block = sourceIter.nextBlock();
 if (block != null) {
-  count++;
+  blockCount++;
 }
   }
+  if (checkblockPoolCount) {
+LOG.info("Block Pool Id:  {}, blockCount: {}", blockPoolID, 
blockCount);
+assertTrue(blockCount > 0);
+  }
+  count += blockCount;
 }
 return count;
   }
@@ -320,10 +333,10 @@ public class DiskBalancerTestUtil {
dnNode.getFSDataset().getFsVolumeReferences()) {
 source = (FsVolumeImpl) refs.get(0);
 dest = (FsVolumeIm

hadoop git commit: HDFS-13188. Disk Balancer: Support multiple block pools during block move. Contributed by Bharat Viswanadham.

2018-03-06 Thread inigoiri 
Repository: hadoop
Updated Branches:
  refs/heads/trunk e6f99e205 -> 706072566


HDFS-13188. Disk Balancer: Support multiple block pools during block move. 
Contributed by Bharat Viswanadham.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/70607256
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/70607256
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/70607256

Branch: refs/heads/trunk
Commit: 7060725662cb3317ff2f0fcc38f965fd23e8e6aa
Parents: e6f99e2
Author: Inigo Goiri 
Authored: Tue Mar 6 09:09:32 2018 -0800
Committer: Inigo Goiri 
Committed: Tue Mar 6 09:09:32 2018 -0800

--
 .../hdfs/server/datanode/DiskBalancer.java  |  4 +-
 .../diskbalancer/DiskBalancerTestUtil.java  | 21 --
 .../server/diskbalancer/TestDiskBalancer.java   | 72 +---
 .../diskbalancer/TestDiskBalancerRPC.java   |  4 +-
 4 files changed, 85 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/70607256/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
index e90a47e..91c3624 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
@@ -958,8 +958,8 @@ public class DiskBalancer {
   ExtendedBlock block = null;
   while (block == null && currentCount < poolIters.size()) {
 currentCount++;
-poolIndex = poolIndex++ % poolIters.size();
-FsVolumeSpi.BlockIterator currentPoolIter = poolIters.get(poolIndex);
+int index = poolIndex++ % poolIters.size();
+FsVolumeSpi.BlockIterator currentPoolIter = poolIters.get(index);
 block = getBlockToCopy(currentPoolIter, item);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/70607256/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java
index bc4181b..bd8dbce 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java
@@ -40,6 +40,10 @@ import 
org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolume;
 import 
org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolumeSet;
 import org.apache.hadoop.util.Time;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
@@ -51,6 +55,7 @@ import java.util.concurrent.TimeoutException;
  * Helper class to create various cluster configurations at run time.
  */
 public class DiskBalancerTestUtil {
+  static final Logger LOG = LoggerFactory.getLogger(TestDiskBalancer.class);
   public static final long MB = 1024 * 1024L;
   public static final long GB = MB * 1024L;
   public static final long TB = GB * 1024L;
@@ -241,17 +246,25 @@ public class DiskBalancerTestUtil {
* @return Number of Blocks.
* @throws IOException
*/
-  public static int getBlockCount(FsVolumeSpi source) throws IOException {
+  public static int getBlockCount(FsVolumeSpi source,
+  boolean checkblockPoolCount)
+  throws IOException {
 int count = 0;
 for (String blockPoolID : source.getBlockPoolList()) {
   FsVolumeSpi.BlockIterator sourceIter =
   source.newBlockIterator(blockPoolID, "TestDiskBalancerSource");
+  int blockCount = 0;
   while (!sourceIter.atEnd()) {
 ExtendedBlock block = sourceIter.nextBlock();
 if (block != null) {
-  count++;
+  blockCount++;
 }
   }
+  if (checkblockPoolCount) {
+LOG.info("Block Pool Id:  {}, blockCount: {}", blockPoolID, 
blockCount);
+assertTrue(blockCount > 0);
+  }
+  count += blockCount;
 }
 return count;
   }
@@ -320,10 +333,10 @@ public class DiskBalancerTestUtil {
dnNode.getFSDataset().getFsVolumeReferences()