[04/50] [abbrv] hadoop git commit: HDFS-8684. Erasure Coding: fix some block number calculation for striped block. (yliu)

2015-08-20 Thread zhz 
HDFS-8684. Erasure Coding: fix some block number calculation for striped block. 
(yliu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee01a095
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee01a095
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee01a095

Branch: refs/heads/HDFS-7285-merge
Commit: ee01a09500224136464f2c3e0a5d9ba53242d93f
Parents: 0b7af27
Author: yliu y...@apache.org
Authored: Mon Jul 6 19:14:18 2015 +0800
Committer: yliu y...@apache.org
Committed: Mon Jul 6 19:14:18 2015 +0800

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt |  5 -
 .../hadoop/hdfs/server/blockmanagement/BlockManager.java | 11 ---
 .../apache/hadoop/hdfs/server/namenode/INodeFile.java|  6 +-
 3 files changed, 9 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee01a095/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 7e10753..8f720fc 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -325,4 +325,7 @@
 multiple times.  (Kai Sasaki via szetszwo)
 
 HDFS-8468. 2 RPC calls for every file read in DFSClient#open(..) resulting 
in
-double Audit log entries (vinayakumarb)
\ No newline at end of file
+double Audit log entries (vinayakumarb)
+
+HDFS-8684. Erasure Coding: fix some block number calculation for striped
+block. (yliu)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee01a095/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index dcb0be77..c98bdb0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3109,12 +3109,12 @@ public class BlockManager {
 bc.getStoragePolicyID());
 final ListStorageType excessTypes = storagePolicy.chooseExcess(
 replication, DatanodeStorageInfo.toStorageTypes(nonExcess));
-if (!storedBlock.isStriped()) {
-  chooseExcessReplicasContiguous(bc, nonExcess, storedBlock,
-  replication, addedNode, delNodeHint, excessTypes);
-} else {
+if (storedBlock.isStriped()) {
   chooseExcessReplicasStriped(bc, nonExcess, storedBlock, delNodeHint,
   excessTypes);
+} else {
+  chooseExcessReplicasContiguous(bc, nonExcess, storedBlock,
+  replication, addedNode, delNodeHint, excessTypes);
 }
   }
 
@@ -3191,9 +3191,6 @@ public class BlockManager {
 assert storedBlock instanceof BlockInfoStriped;
 BlockInfoStriped sblk = (BlockInfoStriped) storedBlock;
 short groupSize = sblk.getTotalBlockNum();
-if (nonExcess.size() = groupSize) {
-  return;
-}
 BlockPlacementPolicy placementPolicy = placementPolicies.getPolicy(true);
 ListDatanodeStorageInfo empty = new ArrayList(0);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee01a095/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 4688001..b2b0fea 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -809,13 +809,9 @@ public class INodeFile extends INodeWithAdditionalFields
   if (!includesLastUcBlock) {
 size = 0;
   } else if (usePreferredBlockSize4LastUcBlock) {
-// Striped blocks keeps block group which counts
-// (data blocks num + parity blocks num). When you
-// count actual used size by BlockInfoStripedUC must
-// be multiplied by these blocks number.
 BlockInfoStripedUnderConstruction blockInfoStripedUC
 = (BlockInfoStripedUnderConstruction) blockInfos[last];
-size = getPreferredBlockSize() *

[26/50] hadoop git commit: HDFS-8684. Erasure Coding: fix some block number calculation for striped block. (yliu)

2015-08-14 Thread vinayakumarb 
HDFS-8684. Erasure Coding: fix some block number calculation for striped block. 
(yliu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/904bd7d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/904bd7d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/904bd7d2

Branch: refs/heads/HDFS-7285-REBASE
Commit: 904bd7d25b3c51265a6286f0a27031642194500b
Parents: 185b173
Author: yliu y...@apache.org
Authored: Mon Jul 6 19:14:18 2015 +0800
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Thu Aug 13 17:16:20 2015 +0530

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt |  5 -
 .../hadoop/hdfs/server/blockmanagement/BlockManager.java | 11 ---
 .../apache/hadoop/hdfs/server/namenode/INodeFile.java|  6 +-
 3 files changed, 9 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/904bd7d2/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 7e10753..8f720fc 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -325,4 +325,7 @@
 multiple times.  (Kai Sasaki via szetszwo)
 
 HDFS-8468. 2 RPC calls for every file read in DFSClient#open(..) resulting 
in
-double Audit log entries (vinayakumarb)
\ No newline at end of file
+double Audit log entries (vinayakumarb)
+
+HDFS-8684. Erasure Coding: fix some block number calculation for striped
+block. (yliu)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/904bd7d2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 66fca2f..d951d3f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3155,12 +3155,12 @@ public class BlockManager implements BlockStatsMXBean {
 bc.getStoragePolicyID());
 final ListStorageType excessTypes = storagePolicy.chooseExcess(
 replication, DatanodeStorageInfo.toStorageTypes(nonExcess));
-if (!storedBlock.isStriped()) {
-  chooseExcessReplicasContiguous(bc, nonExcess, storedBlock,
-  replication, addedNode, delNodeHint, excessTypes);
-} else {
+if (storedBlock.isStriped()) {
   chooseExcessReplicasStriped(bc, nonExcess, storedBlock, delNodeHint,
   excessTypes);
+} else {
+  chooseExcessReplicasContiguous(bc, nonExcess, storedBlock,
+  replication, addedNode, delNodeHint, excessTypes);
 }
   }
 
@@ -3237,9 +3237,6 @@ public class BlockManager implements BlockStatsMXBean {
 assert storedBlock instanceof BlockInfoStriped;
 BlockInfoStriped sblk = (BlockInfoStriped) storedBlock;
 short groupSize = sblk.getTotalBlockNum();
-if (nonExcess.size() = groupSize) {
-  return;
-}
 BlockPlacementPolicy placementPolicy = placementPolicies.getPolicy(true);
 ListDatanodeStorageInfo empty = new ArrayList(0);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/904bd7d2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 05425bf..c8943ab 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -814,13 +814,9 @@ public class INodeFile extends INodeWithAdditionalFields
   if (!includesLastUcBlock) {
 size = 0;
   } else if (usePreferredBlockSize4LastUcBlock) {
-// Striped blocks keeps block group which counts
-// (data blocks num + parity blocks num). When you
-// count actual used size by BlockInfoStripedUC must
-// be multiplied by these blocks number.
 BlockInfoStripedUnderConstruction blockInfoStripedUC
 =

hadoop git commit: HDFS-8684. Erasure Coding: fix some block number calculation for striped block. (yliu)

2015-07-06 Thread yliu 
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7285 0b7af27b9 - ee01a0950


HDFS-8684. Erasure Coding: fix some block number calculation for striped block. 
(yliu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee01a095
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee01a095
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee01a095

Branch: refs/heads/HDFS-7285
Commit: ee01a09500224136464f2c3e0a5d9ba53242d93f
Parents: 0b7af27
Author: yliu y...@apache.org
Authored: Mon Jul 6 19:14:18 2015 +0800
Committer: yliu y...@apache.org
Committed: Mon Jul 6 19:14:18 2015 +0800

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt |  5 -
 .../hadoop/hdfs/server/blockmanagement/BlockManager.java | 11 ---
 .../apache/hadoop/hdfs/server/namenode/INodeFile.java|  6 +-
 3 files changed, 9 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee01a095/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 7e10753..8f720fc 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -325,4 +325,7 @@
 multiple times.  (Kai Sasaki via szetszwo)
 
 HDFS-8468. 2 RPC calls for every file read in DFSClient#open(..) resulting 
in
-double Audit log entries (vinayakumarb)
\ No newline at end of file
+double Audit log entries (vinayakumarb)
+
+HDFS-8684. Erasure Coding: fix some block number calculation for striped
+block. (yliu)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee01a095/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index dcb0be77..c98bdb0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3109,12 +3109,12 @@ public class BlockManager {
 bc.getStoragePolicyID());
 final ListStorageType excessTypes = storagePolicy.chooseExcess(
 replication, DatanodeStorageInfo.toStorageTypes(nonExcess));
-if (!storedBlock.isStriped()) {
-  chooseExcessReplicasContiguous(bc, nonExcess, storedBlock,
-  replication, addedNode, delNodeHint, excessTypes);
-} else {
+if (storedBlock.isStriped()) {
   chooseExcessReplicasStriped(bc, nonExcess, storedBlock, delNodeHint,
   excessTypes);
+} else {
+  chooseExcessReplicasContiguous(bc, nonExcess, storedBlock,
+  replication, addedNode, delNodeHint, excessTypes);
 }
   }
 
@@ -3191,9 +3191,6 @@ public class BlockManager {
 assert storedBlock instanceof BlockInfoStriped;
 BlockInfoStriped sblk = (BlockInfoStriped) storedBlock;
 short groupSize = sblk.getTotalBlockNum();
-if (nonExcess.size() = groupSize) {
-  return;
-}
 BlockPlacementPolicy placementPolicy = placementPolicies.getPolicy(true);
 ListDatanodeStorageInfo empty = new ArrayList(0);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee01a095/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 4688001..b2b0fea 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -809,13 +809,9 @@ public class INodeFile extends INodeWithAdditionalFields
   if (!includesLastUcBlock) {
 size = 0;
   } else if (usePreferredBlockSize4LastUcBlock) {
-// Striped blocks keeps block group which counts
-// (data blocks num + parity blocks num). When you
-// count actual used size by BlockInfoStripedUC must
-// be multiplied by these blocks number.
 BlockInfoStripedUnderConstruction blockInfoStripedUC
 =