[02/49] hadoop git commit: HDFS-9709. DiskBalancer : Add tests for disk balancer using a Mock Mover class. Contributed by Anu Engineer.

2016-06-24 Thread arp 
HDFS-9709. DiskBalancer : Add tests for disk balancer using a Mock Mover class. 
Contributed by Anu Engineer.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c606bf5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c606bf5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c606bf5

Branch: refs/heads/trunk
Commit: 6c606bf5c8c1ace381ce73679c2be96d5475ba34
Parents: 9be9703
Author: Anu Engineer 
Authored: Tue Mar 22 16:26:49 2016 -0700
Committer: Arpit Agarwal 
Committed: Thu Jun 23 18:18:48 2016 -0700

--
 .../hdfs/server/datanode/DiskBalancer.java  |  14 +-
 .../DiskBalancerResultVerifier.java |  42 ++
 .../diskbalancer/TestDiskBalancerRPC.java   |  39 +-
 .../TestDiskBalancerWithMockMover.java  | 570 +++
 4 files changed, 628 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c606bf5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
index d1bc1f1..972f0fc 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
@@ -40,7 +40,11 @@ import java.io.IOException;
 import java.nio.charset.Charset;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.concurrent.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantLock;
 
 /**
@@ -106,6 +110,7 @@ public class DiskBalancer {
   this.isDiskBalancerEnabled = false;
   this.currentResult = Result.NO_PLAN;
   if ((this.future != null) && (!this.future.isDone())) {
+this.currentResult = Result.PLAN_CANCELLED;
 this.blockMover.setExitFlag();
 shutdownExecutor();
   }
@@ -120,9 +125,9 @@ public class DiskBalancer {
   private void shutdownExecutor() {
 scheduler.shutdown();
 try {
-  if(!scheduler.awaitTermination(30, TimeUnit.SECONDS)) {
+  if(!scheduler.awaitTermination(10, TimeUnit.SECONDS)) {
 scheduler.shutdownNow();
-if (!scheduler.awaitTermination(30, TimeUnit.SECONDS)) {
+if (!scheduler.awaitTermination(10, TimeUnit.SECONDS)) {
   LOG.error("Disk Balancer : Scheduler did not terminate.");
 }
   }
@@ -218,6 +223,7 @@ public class DiskBalancer {
   if (!this.future.isDone()) {
 this.blockMover.setExitFlag();
 shutdownExecutor();
+this.currentResult = Result.PLAN_CANCELLED;
   }
 } finally {
   lock.unlock();
@@ -537,7 +543,7 @@ public class DiskBalancer {
   /**
* Holds references to actual volumes that we will be operating against.
*/
-  static class VolumePair {
+  public static class VolumePair {
 private final FsVolumeSpi source;
 private final FsVolumeSpi dest;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c606bf5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerResultVerifier.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerResultVerifier.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerResultVerifier.java
new file mode 100644
index 000..5abb33c
--- /dev/null
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerResultVerifier.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express

[07/49] hadoop git commit: HDFS-9709. DiskBalancer : Add tests for disk balancer using a Mock Mover class. Contributed by Anu Engineer.

2016-06-23 Thread arp 
HDFS-9709. DiskBalancer : Add tests for disk balancer using a Mock Mover class. 
Contributed by Anu Engineer.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c606bf5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c606bf5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c606bf5

Branch: refs/heads/HDFS-1312
Commit: 6c606bf5c8c1ace381ce73679c2be96d5475ba34
Parents: 9be9703
Author: Anu Engineer 
Authored: Tue Mar 22 16:26:49 2016 -0700
Committer: Arpit Agarwal 
Committed: Thu Jun 23 18:18:48 2016 -0700

--
 .../hdfs/server/datanode/DiskBalancer.java  |  14 +-
 .../DiskBalancerResultVerifier.java |  42 ++
 .../diskbalancer/TestDiskBalancerRPC.java   |  39 +-
 .../TestDiskBalancerWithMockMover.java  | 570 +++
 4 files changed, 628 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c606bf5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
index d1bc1f1..972f0fc 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
@@ -40,7 +40,11 @@ import java.io.IOException;
 import java.nio.charset.Charset;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.concurrent.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantLock;
 
 /**
@@ -106,6 +110,7 @@ public class DiskBalancer {
   this.isDiskBalancerEnabled = false;
   this.currentResult = Result.NO_PLAN;
   if ((this.future != null) && (!this.future.isDone())) {
+this.currentResult = Result.PLAN_CANCELLED;
 this.blockMover.setExitFlag();
 shutdownExecutor();
   }
@@ -120,9 +125,9 @@ public class DiskBalancer {
   private void shutdownExecutor() {
 scheduler.shutdown();
 try {
-  if(!scheduler.awaitTermination(30, TimeUnit.SECONDS)) {
+  if(!scheduler.awaitTermination(10, TimeUnit.SECONDS)) {
 scheduler.shutdownNow();
-if (!scheduler.awaitTermination(30, TimeUnit.SECONDS)) {
+if (!scheduler.awaitTermination(10, TimeUnit.SECONDS)) {
   LOG.error("Disk Balancer : Scheduler did not terminate.");
 }
   }
@@ -218,6 +223,7 @@ public class DiskBalancer {
   if (!this.future.isDone()) {
 this.blockMover.setExitFlag();
 shutdownExecutor();
+this.currentResult = Result.PLAN_CANCELLED;
   }
 } finally {
   lock.unlock();
@@ -537,7 +543,7 @@ public class DiskBalancer {
   /**
* Holds references to actual volumes that we will be operating against.
*/
-  static class VolumePair {
+  public static class VolumePair {
 private final FsVolumeSpi source;
 private final FsVolumeSpi dest;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c606bf5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerResultVerifier.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerResultVerifier.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerResultVerifier.java
new file mode 100644
index 000..5abb33c
--- /dev/null
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerResultVerifier.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either

hadoop git commit: HDFS-9709. DiskBalancer : Add tests for disk balancer using a Mock Mover class. Contributed by Anu Engineer.

2016-03-22 Thread aengineer 
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-1312 c87c8458d -> 3258c8b03


HDFS-9709. DiskBalancer : Add tests for disk balancer using a Mock Mover class. 
Contributed by Anu Engineer.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3258c8b0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3258c8b0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3258c8b0

Branch: refs/heads/HDFS-1312
Commit: 3258c8b034a38a1e684164e12ef7da00263edbc0
Parents: c87c845
Author: Anu Engineer 
Authored: Tue Mar 22 16:26:49 2016 -0700
Committer: Anu Engineer 
Committed: Tue Mar 22 16:26:49 2016 -0700

--
 .../hdfs/server/datanode/DiskBalancer.java  |  14 +-
 .../DiskBalancerResultVerifier.java |  42 ++
 .../diskbalancer/TestDiskBalancerRPC.java   |  39 +-
 .../TestDiskBalancerWithMockMover.java  | 570 +++
 4 files changed, 628 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3258c8b0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
index d1bc1f1..972f0fc 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
@@ -40,7 +40,11 @@ import java.io.IOException;
 import java.nio.charset.Charset;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.concurrent.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantLock;
 
 /**
@@ -106,6 +110,7 @@ public class DiskBalancer {
   this.isDiskBalancerEnabled = false;
   this.currentResult = Result.NO_PLAN;
   if ((this.future != null) && (!this.future.isDone())) {
+this.currentResult = Result.PLAN_CANCELLED;
 this.blockMover.setExitFlag();
 shutdownExecutor();
   }
@@ -120,9 +125,9 @@ public class DiskBalancer {
   private void shutdownExecutor() {
 scheduler.shutdown();
 try {
-  if(!scheduler.awaitTermination(30, TimeUnit.SECONDS)) {
+  if(!scheduler.awaitTermination(10, TimeUnit.SECONDS)) {
 scheduler.shutdownNow();
-if (!scheduler.awaitTermination(30, TimeUnit.SECONDS)) {
+if (!scheduler.awaitTermination(10, TimeUnit.SECONDS)) {
   LOG.error("Disk Balancer : Scheduler did not terminate.");
 }
   }
@@ -218,6 +223,7 @@ public class DiskBalancer {
   if (!this.future.isDone()) {
 this.blockMover.setExitFlag();
 shutdownExecutor();
+this.currentResult = Result.PLAN_CANCELLED;
   }
 } finally {
   lock.unlock();
@@ -537,7 +543,7 @@ public class DiskBalancer {
   /**
* Holds references to actual volumes that we will be operating against.
*/
-  static class VolumePair {
+  public static class VolumePair {
 private final FsVolumeSpi source;
 private final FsVolumeSpi dest;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3258c8b0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerResultVerifier.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerResultVerifier.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerResultVerifier.java
new file mode 100644
index 000..5abb33c
--- /dev/null
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerResultVerifier.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is