[22/34] git commit: HDFS-6932. Balancer and Mover tools should ignore replicas on RAM_DISK. (Contributed by Xiaoyu Yao)

2014-10-17 Thread jitendra 
HDFS-6932. Balancer and Mover tools should ignore replicas on RAM_DISK. 
(Contributed by Xiaoyu Yao)

Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69828a9b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69828a9b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69828a9b

Branch: refs/heads/branch-2
Commit: 69828a9bf0b84e43f807b51638cf080d5a5745d5
Parents: 70f70f8
Author: arp a...@apache.org
Authored: Wed Sep 24 21:08:59 2014 -0700
Committer: Jitendra Pandey Jitendra@Jitendra-Pandeys-MacBook-Pro-4.local
Committed: Fri Oct 17 13:42:02 2014 -0700

--
 .../org/apache/hadoop/hdfs/StorageType.java | 31 +--
 .../hadoop/hdfs/server/balancer/Balancer.java   |  2 +-
 .../apache/hadoop/hdfs/server/mover/Mover.java  | 30 --
 .../org/apache/hadoop/hdfs/DFSTestUtil.java | 33 +++
 .../org/apache/hadoop/hdfs/MiniDFSCluster.java  |  3 +-
 .../hdfs/server/balancer/TestBalancer.java  | 96 +--
 .../hdfs/server/mover/TestStorageMover.java | 97 +++-
 7 files changed, 265 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/69828a9b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
index c11d4e6..99cae9a 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hdfs;
 
 import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -31,18 +32,34 @@ import org.apache.hadoop.classification.InterfaceStability;
 @InterfaceAudience.Public
 @InterfaceStability.Unstable
 public enum StorageType {
-  DISK,
-  SSD,
-  ARCHIVE,
-  RAM_DISK;
+  DISK(false),
+  SSD(false),
+  ARCHIVE(false),
+  RAM_DISK(true);
+
+  private final boolean isTransient;
 
   public static final StorageType DEFAULT = DISK;
-  
+
   public static final StorageType[] EMPTY_ARRAY = {};
-  
+
   private static final StorageType[] VALUES = values();
-  
+
+  StorageType(boolean isTransient) { this.isTransient = isTransient; }
+
+  public boolean isMovable() { return isTransient == false; }
+
   public static ListStorageType asList() {
 return Arrays.asList(VALUES);
   }
+
+  public static ListStorageType getMovableTypes() {
+ListStorageType movableTypes = new ArrayListStorageType();
+for (StorageType t : VALUES) {
+  if ( t.isTransient == false ) {
+movableTypes.add(t);
+  }
+}
+return movableTypes;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/69828a9b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index d77da7c..87348b3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -272,7 +272,7 @@ public class Balancer {
 long overLoadedBytes = 0L, underLoadedBytes = 0L;
 for(DatanodeStorageReport r : reports) {
   final DDatanode dn = dispatcher.newDatanode(r.getDatanodeInfo());
-  for(StorageType t : StorageType.asList()) {
+  for(StorageType t : StorageType.getMovableTypes()) {
 final Double utilization = policy.getUtilization(r, t);
 if (utilization == null) { // datanode does not have such storage type 
   continue;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/69828a9b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java

[28/50] [abbrv] git commit: HDFS-6932. Balancer and Mover tools should ignore replicas on RAM_DISK. (Contributed by Xiaoyu Yao)

2014-10-01 Thread arp 
HDFS-6932. Balancer and Mover tools should ignore replicas on RAM_DISK. 
(Contributed by Xiaoyu Yao)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8e7fbe8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8e7fbe8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8e7fbe8

Branch: refs/heads/trunk
Commit: e8e7fbe81abc64a9ae3d2f3f62c088426073b2bf
Parents: feda473
Author: arp a...@apache.org
Authored: Wed Sep 24 21:08:59 2014 -0700
Committer: arp a...@apache.org
Committed: Wed Sep 24 21:08:59 2014 -0700

--
 .../hadoop-hdfs/CHANGES-HDFS-6581.txt   |  4 +
 .../org/apache/hadoop/hdfs/StorageType.java | 31 +--
 .../hadoop/hdfs/server/balancer/Balancer.java   |  2 +-
 .../apache/hadoop/hdfs/server/mover/Mover.java  | 30 --
 .../org/apache/hadoop/hdfs/DFSTestUtil.java | 33 +++
 .../org/apache/hadoop/hdfs/MiniDFSCluster.java  |  3 +-
 .../hdfs/server/balancer/TestBalancer.java  | 96 ++--
 .../hdfs/server/mover/TestStorageMover.java | 89 --
 8 files changed, 257 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8e7fbe8/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
index 6eb8cec..7693981 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
@@ -74,3 +74,7 @@
 HDFS-7143. Fix findbugs warnings in HDFS-6581 branch. (szetszwo via
 Arpit Agarwal)
 
+HDFS-6932. Balancer and Mover tools should ignore replicas on RAM_DISK.
+(Xiaoyu Yao via Arpit Agarwal)
+
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8e7fbe8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
index bcbe5c0..99cae9a 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hdfs;
 
 import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -31,18 +32,34 @@ import org.apache.hadoop.classification.InterfaceStability;
 @InterfaceAudience.Public
 @InterfaceStability.Unstable
 public enum StorageType {
-  DISK,
-  SSD,
-  ARCHIVE,
-  RAM_DISK;  
+  DISK(false),
+  SSD(false),
+  ARCHIVE(false),
+  RAM_DISK(true);
+
+  private final boolean isTransient;
 
   public static final StorageType DEFAULT = DISK;
-  
+
   public static final StorageType[] EMPTY_ARRAY = {};
-  
+
   private static final StorageType[] VALUES = values();
-  
+
+  StorageType(boolean isTransient) { this.isTransient = isTransient; }
+
+  public boolean isMovable() { return isTransient == false; }
+
   public static ListStorageType asList() {
 return Arrays.asList(VALUES);
   }
+
+  public static ListStorageType getMovableTypes() {
+ListStorageType movableTypes = new ArrayListStorageType();
+for (StorageType t : VALUES) {
+  if ( t.isTransient == false ) {
+movableTypes.add(t);
+  }
+}
+return movableTypes;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8e7fbe8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 67994c8..2a19537 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -271,7 +271,7 @@ public class Balancer {
 long overLoadedBytes = 0L, underLoadedBytes = 0L;
 for(DatanodeStorageReport r : reports) {
   final DDatanode dn = dispatcher.newDatanode(r.getDatanodeInfo());
-  for(StorageType t : StorageType.asList()) {
+  for(StorageType t : StorageType.getMovableTypes()) {
 final Double utilization = policy.getUtilization(r, t);
 if (utilization == null) { // datanode does not have such storage type 
   continue;

git commit: HDFS-6932. Balancer and Mover tools should ignore replicas on RAM_DISK. (Contributed by Xiaoyu Yao)

2014-09-24 Thread arp 
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-6581 feda4733a - e8e7fbe81


HDFS-6932. Balancer and Mover tools should ignore replicas on RAM_DISK. 
(Contributed by Xiaoyu Yao)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8e7fbe8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8e7fbe8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8e7fbe8

Branch: refs/heads/HDFS-6581
Commit: e8e7fbe81abc64a9ae3d2f3f62c088426073b2bf
Parents: feda473
Author: arp a...@apache.org
Authored: Wed Sep 24 21:08:59 2014 -0700
Committer: arp a...@apache.org
Committed: Wed Sep 24 21:08:59 2014 -0700

--
 .../hadoop-hdfs/CHANGES-HDFS-6581.txt   |  4 +
 .../org/apache/hadoop/hdfs/StorageType.java | 31 +--
 .../hadoop/hdfs/server/balancer/Balancer.java   |  2 +-
 .../apache/hadoop/hdfs/server/mover/Mover.java  | 30 --
 .../org/apache/hadoop/hdfs/DFSTestUtil.java | 33 +++
 .../org/apache/hadoop/hdfs/MiniDFSCluster.java  |  3 +-
 .../hdfs/server/balancer/TestBalancer.java  | 96 ++--
 .../hdfs/server/mover/TestStorageMover.java | 89 --
 8 files changed, 257 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8e7fbe8/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
index 6eb8cec..7693981 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
@@ -74,3 +74,7 @@
 HDFS-7143. Fix findbugs warnings in HDFS-6581 branch. (szetszwo via
 Arpit Agarwal)
 
+HDFS-6932. Balancer and Mover tools should ignore replicas on RAM_DISK.
+(Xiaoyu Yao via Arpit Agarwal)
+
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8e7fbe8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
index bcbe5c0..99cae9a 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hdfs;
 
 import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -31,18 +32,34 @@ import org.apache.hadoop.classification.InterfaceStability;
 @InterfaceAudience.Public
 @InterfaceStability.Unstable
 public enum StorageType {
-  DISK,
-  SSD,
-  ARCHIVE,
-  RAM_DISK;  
+  DISK(false),
+  SSD(false),
+  ARCHIVE(false),
+  RAM_DISK(true);
+
+  private final boolean isTransient;
 
   public static final StorageType DEFAULT = DISK;
-  
+
   public static final StorageType[] EMPTY_ARRAY = {};
-  
+
   private static final StorageType[] VALUES = values();
-  
+
+  StorageType(boolean isTransient) { this.isTransient = isTransient; }
+
+  public boolean isMovable() { return isTransient == false; }
+
   public static ListStorageType asList() {
 return Arrays.asList(VALUES);
   }
+
+  public static ListStorageType getMovableTypes() {
+ListStorageType movableTypes = new ArrayListStorageType();
+for (StorageType t : VALUES) {
+  if ( t.isTransient == false ) {
+movableTypes.add(t);
+  }
+}
+return movableTypes;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8e7fbe8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 67994c8..2a19537 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -271,7 +271,7 @@ public class Balancer {
 long overLoadedBytes = 0L, underLoadedBytes = 0L;
 for(DatanodeStorageReport r : reports) {
   final DDatanode dn = dispatcher.newDatanode(r.getDatanodeInfo());
-  for(StorageType t : StorageType.asList()) {
+  for(StorageType t : StorageType.getMovableTypes()) {
 final Double utilization = policy.getUtilization(r, t);
 if