HDFS-8684. Erasure Coding: fix some block number calculation for striped block. (yliu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee01a095 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee01a095 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee01a095 Branch: refs/heads/HDFS-7285-merge Commit: ee01a09500224136464f2c3e0a5d9ba53242d93f Parents: 0b7af27 Author: yliu <y...@apache.org> Authored: Mon Jul 6 19:14:18 2015 +0800 Committer: yliu <y...@apache.org> Committed: Mon Jul 6 19:14:18 2015 +0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 5 ++++- .../hadoop/hdfs/server/blockmanagement/BlockManager.java | 11 ++++------- .../apache/hadoop/hdfs/server/namenode/INodeFile.java | 6 +----- 3 files changed, 9 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee01a095/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 7e10753..8f720fc 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -325,4 +325,7 @@ multiple times. (Kai Sasaki via szetszwo) HDFS-8468. 2 RPC calls for every file read in DFSClient#open(..) resulting in - double Audit log entries (vinayakumarb) \ No newline at end of file + double Audit log entries (vinayakumarb) + + HDFS-8684. Erasure Coding: fix some block number calculation for striped + block. (yliu) http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee01a095/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index dcb0be77..c98bdb0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -3109,12 +3109,12 @@ public class BlockManager { bc.getStoragePolicyID()); final List<StorageType> excessTypes = storagePolicy.chooseExcess( replication, DatanodeStorageInfo.toStorageTypes(nonExcess)); - if (!storedBlock.isStriped()) { - chooseExcessReplicasContiguous(bc, nonExcess, storedBlock, - replication, addedNode, delNodeHint, excessTypes); - } else { + if (storedBlock.isStriped()) { chooseExcessReplicasStriped(bc, nonExcess, storedBlock, delNodeHint, excessTypes); + } else { + chooseExcessReplicasContiguous(bc, nonExcess, storedBlock, + replication, addedNode, delNodeHint, excessTypes); } } @@ -3191,9 +3191,6 @@ public class BlockManager { assert storedBlock instanceof BlockInfoStriped; BlockInfoStriped sblk = (BlockInfoStriped) storedBlock; short groupSize = sblk.getTotalBlockNum(); - if (nonExcess.size() <= groupSize) { - return; - } BlockPlacementPolicy placementPolicy = placementPolicies.getPolicy(true); List<DatanodeStorageInfo> empty = new ArrayList<>(0); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee01a095/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 4688001..b2b0fea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -809,13 +809,9 @@ public class INodeFile extends INodeWithAdditionalFields if (!includesLastUcBlock) { size = 0; } else if (usePreferredBlockSize4LastUcBlock) { - // Striped blocks keeps block group which counts - // (data blocks num + parity blocks num). When you - // count actual used size by BlockInfoStripedUC must - // be multiplied by these blocks number. BlockInfoStripedUnderConstruction blockInfoStripedUC = (BlockInfoStripedUnderConstruction) blockInfos[last]; - size = getPreferredBlockSize() * blockInfoStripedUC.getTotalBlockNum(); + size = getPreferredBlockSize() * blockInfoStripedUC.getDataBlockNum(); } } //sum other blocks
