[11/51] [abbrv] hadoop git commit: HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang
HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e39e00c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e39e00c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e39e00c Branch: refs/heads/HDFS-7285 Commit: 7e39e00c1012b877a7b8b29adf8e5a7250f542b5 Parents: 2b2859e Author: Zhe Zhang z...@apache.org Authored: Mon Feb 9 10:27:14 2015 -0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:01:04 2015 -0700 -- .../server/blockmanagement/BlockIdManager.java| 8 .../hdfs/server/blockmanagement/BlockManager.java | 18 +- 2 files changed, 21 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e39e00c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java index c8b9d20..e7f8a05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java @@ -211,4 +211,12 @@ public class BlockIdManager { .LAST_RESERVED_BLOCK_ID); generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP; } + + public static boolean isStripedBlockID(long id) { +return id 0; + } + + public static long convertToGroupID(long id) { +return id (~(HdfsConstants.MAX_BLOCKS_IN_GROUP - 1)); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e39e00c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 524afa0..97dd2c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1925,7 +1925,7 @@ public class BlockManager { break; } -BlockInfoContiguous bi = blocksMap.getStoredBlock(b); +BlockInfoContiguous bi = getStoredBlock(b); if (bi == null) { if (LOG.isDebugEnabled()) { LOG.debug(BLOCK* rescanPostponedMisreplicatedBlocks: + @@ -2068,7 +2068,7 @@ public class BlockManager { continue; } - BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(iblk); + BlockInfoContiguous storedBlock = getStoredBlock(iblk); // If block does not belong to any file, we are done. if (storedBlock == null) continue; @@ -2208,7 +2208,7 @@ public class BlockManager { } // find block by blockId -BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(block); +BlockInfoContiguous storedBlock = getStoredBlock(block); if(storedBlock == null) { // If blocksMap does not contain reported block id, // the replica should be removed from the data-node. @@ -2499,7 +2499,7 @@ public class BlockManager { DatanodeDescriptor node = storageInfo.getDatanodeDescriptor(); if (block instanceof BlockInfoContiguousUnderConstruction) { //refresh our copy in case the block got completed in another thread - storedBlock = blocksMap.getStoredBlock(block); + storedBlock = getStoredBlock(block); } else { storedBlock = block; } @@ -3361,7 +3361,15 @@ public class BlockManager { } public BlockInfoContiguous getStoredBlock(Block block) { -return blocksMap.getStoredBlock(block); +BlockInfoContiguous info = null; +if (BlockIdManager.isStripedBlockID(block.getBlockId())) { + info = blocksMap.getStoredBlock( + new Block(BlockIdManager.convertToGroupID(block.getBlockId(; +} +if (info == null) { + info = blocksMap.getStoredBlock(block); +} +return info; } /** updates a block in under replication queue */
[16/50] [abbrv] hadoop git commit: HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang
HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cbe6ef9c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cbe6ef9c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cbe6ef9c Branch: refs/heads/HDFS-7285 Commit: cbe6ef9cb5616904c4d6f598e51301a8baa80ab5 Parents: 6fb4ed4 Author: Zhe Zhang z...@apache.org Authored: Mon Feb 9 10:27:14 2015 -0800 Committer: Zhe Zhang z...@apache.org Committed: Thu Apr 2 11:06:03 2015 -0700 -- .../server/blockmanagement/BlockIdManager.java| 8 .../hdfs/server/blockmanagement/BlockManager.java | 18 +- 2 files changed, 21 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbe6ef9c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java index c8b9d20..e7f8a05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java @@ -211,4 +211,12 @@ public class BlockIdManager { .LAST_RESERVED_BLOCK_ID); generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP; } + + public static boolean isStripedBlockID(long id) { +return id 0; + } + + public static long convertToGroupID(long id) { +return id (~(HdfsConstants.MAX_BLOCKS_IN_GROUP - 1)); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbe6ef9c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index d9aee62..ed10a18 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1925,7 +1925,7 @@ public class BlockManager { break; } -BlockInfoContiguous bi = blocksMap.getStoredBlock(b); +BlockInfoContiguous bi = getStoredBlock(b); if (bi == null) { if (LOG.isDebugEnabled()) { LOG.debug(BLOCK* rescanPostponedMisreplicatedBlocks: + @@ -2068,7 +2068,7 @@ public class BlockManager { continue; } - BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(iblk); + BlockInfoContiguous storedBlock = getStoredBlock(iblk); // If block does not belong to any file, we are done. if (storedBlock == null) continue; @@ -2208,7 +2208,7 @@ public class BlockManager { } // find block by blockId -BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(block); +BlockInfoContiguous storedBlock = getStoredBlock(block); if(storedBlock == null) { // If blocksMap does not contain reported block id, // the replica should be removed from the data-node. @@ -2499,7 +2499,7 @@ public class BlockManager { DatanodeDescriptor node = storageInfo.getDatanodeDescriptor(); if (block instanceof BlockInfoContiguousUnderConstruction) { //refresh our copy in case the block got completed in another thread - storedBlock = blocksMap.getStoredBlock(block); + storedBlock = getStoredBlock(block); } else { storedBlock = block; } @@ -3361,7 +3361,15 @@ public class BlockManager { } public BlockInfoContiguous getStoredBlock(Block block) { -return blocksMap.getStoredBlock(block); +BlockInfoContiguous info = null; +if (BlockIdManager.isStripedBlockID(block.getBlockId())) { + info = blocksMap.getStoredBlock( + new Block(BlockIdManager.convertToGroupID(block.getBlockId(; +} +if (info == null) { + info = blocksMap.getStoredBlock(block); +} +return info; } /** updates a block in under replication queue */
[17/50] [abbrv] hadoop git commit: HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang
HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7886ed17 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7886ed17 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7886ed17 Branch: refs/heads/HDFS-7285 Commit: 7886ed1705a6c082475578e33bb9a715ab888b22 Parents: 42e26e2 Author: Zhe Zhang z...@apache.org Authored: Mon Feb 9 10:27:14 2015 -0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Mar 30 10:11:23 2015 -0700 -- .../server/blockmanagement/BlockIdManager.java| 8 .../hdfs/server/blockmanagement/BlockManager.java | 18 +- 2 files changed, 21 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7886ed17/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java index c8b9d20..e7f8a05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java @@ -211,4 +211,12 @@ public class BlockIdManager { .LAST_RESERVED_BLOCK_ID); generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP; } + + public static boolean isStripedBlockID(long id) { +return id 0; + } + + public static long convertToGroupID(long id) { +return id (~(HdfsConstants.MAX_BLOCKS_IN_GROUP - 1)); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7886ed17/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index f6e15a3..3102a08 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1925,7 +1925,7 @@ public class BlockManager { break; } -BlockInfoContiguous bi = blocksMap.getStoredBlock(b); +BlockInfoContiguous bi = getStoredBlock(b); if (bi == null) { if (LOG.isDebugEnabled()) { LOG.debug(BLOCK* rescanPostponedMisreplicatedBlocks: + @@ -2068,7 +2068,7 @@ public class BlockManager { continue; } - BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(iblk); + BlockInfoContiguous storedBlock = getStoredBlock(iblk); // If block does not belong to any file, we are done. if (storedBlock == null) continue; @@ -2208,7 +2208,7 @@ public class BlockManager { } // find block by blockId -BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(block); +BlockInfoContiguous storedBlock = getStoredBlock(block); if(storedBlock == null) { // If blocksMap does not contain reported block id, // the replica should be removed from the data-node. @@ -2499,7 +2499,7 @@ public class BlockManager { DatanodeDescriptor node = storageInfo.getDatanodeDescriptor(); if (block instanceof BlockInfoContiguousUnderConstruction) { //refresh our copy in case the block got completed in another thread - storedBlock = blocksMap.getStoredBlock(block); + storedBlock = getStoredBlock(block); } else { storedBlock = block; } @@ -3362,7 +3362,15 @@ public class BlockManager { } public BlockInfoContiguous getStoredBlock(Block block) { -return blocksMap.getStoredBlock(block); +BlockInfoContiguous info = null; +if (BlockIdManager.isStripedBlockID(block.getBlockId())) { + info = blocksMap.getStoredBlock( + new Block(BlockIdManager.convertToGroupID(block.getBlockId(; +} +if (info == null) { + info = blocksMap.getStoredBlock(block); +} +return info; } /** updates a block in under replication queue */
[20/50] [abbrv] hadoop git commit: HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang
HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee9ce478 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee9ce478 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee9ce478 Branch: refs/heads/HDFS-7285 Commit: ee9ce4787bf1b567f65aab7d7b56e93af4ef31d4 Parents: 6a88a8b Author: Zhe Zhang z...@apache.org Authored: Mon Feb 9 10:27:14 2015 -0800 Committer: Zhe Zhang z...@apache.org Committed: Tue Mar 24 11:16:31 2015 -0700 -- .../server/blockmanagement/BlockIdManager.java| 8 .../hdfs/server/blockmanagement/BlockManager.java | 18 +- 2 files changed, 21 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee9ce478/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java index c8b9d20..e7f8a05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java @@ -211,4 +211,12 @@ public class BlockIdManager { .LAST_RESERVED_BLOCK_ID); generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP; } + + public static boolean isStripedBlockID(long id) { +return id 0; + } + + public static long convertToGroupID(long id) { +return id (~(HdfsConstants.MAX_BLOCKS_IN_GROUP - 1)); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee9ce478/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 91cfead..10afdaf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1925,7 +1925,7 @@ public class BlockManager { break; } -BlockInfoContiguous bi = blocksMap.getStoredBlock(b); +BlockInfoContiguous bi = getStoredBlock(b); if (bi == null) { if (LOG.isDebugEnabled()) { LOG.debug(BLOCK* rescanPostponedMisreplicatedBlocks: + @@ -2068,7 +2068,7 @@ public class BlockManager { continue; } - BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(iblk); + BlockInfoContiguous storedBlock = getStoredBlock(iblk); // If block does not belong to any file, we are done. if (storedBlock == null) continue; @@ -2208,7 +2208,7 @@ public class BlockManager { } // find block by blockId -BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(block); +BlockInfoContiguous storedBlock = getStoredBlock(block); if(storedBlock == null) { // If blocksMap does not contain reported block id, // the replica should be removed from the data-node. @@ -2499,7 +2499,7 @@ public class BlockManager { DatanodeDescriptor node = storageInfo.getDatanodeDescriptor(); if (block instanceof BlockInfoContiguousUnderConstruction) { //refresh our copy in case the block got completed in another thread - storedBlock = blocksMap.getStoredBlock(block); + storedBlock = getStoredBlock(block); } else { storedBlock = block; } @@ -3362,7 +3362,15 @@ public class BlockManager { } public BlockInfoContiguous getStoredBlock(Block block) { -return blocksMap.getStoredBlock(block); +BlockInfoContiguous info = null; +if (BlockIdManager.isStripedBlockID(block.getBlockId())) { + info = blocksMap.getStoredBlock( + new Block(BlockIdManager.convertToGroupID(block.getBlockId(; +} +if (info == null) { + info = blocksMap.getStoredBlock(block); +} +return info; } /** updates a block in under replication queue */
[24/50] [abbrv] hadoop git commit: HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang
HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/acc8e001 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/acc8e001 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/acc8e001 Branch: refs/heads/HDFS-7285 Commit: acc8e001f2404cf2e50e65e3bb5bcc59dc77e79f Parents: 54c6526 Author: Zhe Zhang z...@apache.org Authored: Mon Feb 9 10:27:14 2015 -0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Mar 23 11:06:44 2015 -0700 -- .../server/blockmanagement/BlockIdManager.java| 8 .../hdfs/server/blockmanagement/BlockManager.java | 18 +- 2 files changed, 21 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/acc8e001/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java index c8b9d20..e7f8a05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java @@ -211,4 +211,12 @@ public class BlockIdManager { .LAST_RESERVED_BLOCK_ID); generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP; } + + public static boolean isStripedBlockID(long id) { +return id 0; + } + + public static long convertToGroupID(long id) { +return id (~(HdfsConstants.MAX_BLOCKS_IN_GROUP - 1)); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/acc8e001/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 674c0ea..b53f05e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1874,7 +1874,7 @@ public class BlockManager { break; } -BlockInfoContiguous bi = blocksMap.getStoredBlock(b); +BlockInfoContiguous bi = getStoredBlock(b); if (bi == null) { if (LOG.isDebugEnabled()) { LOG.debug(BLOCK* rescanPostponedMisreplicatedBlocks: + @@ -2017,7 +2017,7 @@ public class BlockManager { continue; } - BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(iblk); + BlockInfoContiguous storedBlock = getStoredBlock(iblk); // If block does not belong to any file, we are done. if (storedBlock == null) continue; @@ -2157,7 +2157,7 @@ public class BlockManager { } // find block by blockId -BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(block); +BlockInfoContiguous storedBlock = getStoredBlock(block); if(storedBlock == null) { // If blocksMap does not contain reported block id, // the replica should be removed from the data-node. @@ -2448,7 +2448,7 @@ public class BlockManager { DatanodeDescriptor node = storageInfo.getDatanodeDescriptor(); if (block instanceof BlockInfoContiguousUnderConstruction) { //refresh our copy in case the block got completed in another thread - storedBlock = blocksMap.getStoredBlock(block); + storedBlock = getStoredBlock(block); } else { storedBlock = block; } @@ -3311,7 +3311,15 @@ public class BlockManager { } public BlockInfoContiguous getStoredBlock(Block block) { -return blocksMap.getStoredBlock(block); +BlockInfoContiguous info = null; +if (BlockIdManager.isStripedBlockID(block.getBlockId())) { + info = blocksMap.getStoredBlock( + new Block(BlockIdManager.convertToGroupID(block.getBlockId(; +} +if (info == null) { + info = blocksMap.getStoredBlock(block); +} +return info; } /** updates a block in under replication queue */
[34/50] [abbrv] hadoop git commit: HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang
HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86bc54a5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86bc54a5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86bc54a5 Branch: refs/heads/HDFS-7285 Commit: 86bc54a52b0035a7cb766a3950a93876c74bf426 Parents: 10c794c Author: Zhe Zhang z...@apache.org Authored: Mon Feb 9 10:27:14 2015 -0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Mar 16 13:08:48 2015 -0700 -- .../server/blockmanagement/BlockIdManager.java| 8 .../hdfs/server/blockmanagement/BlockManager.java | 18 +- 2 files changed, 21 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/86bc54a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java index c8b9d20..e7f8a05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java @@ -211,4 +211,12 @@ public class BlockIdManager { .LAST_RESERVED_BLOCK_ID); generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP; } + + public static boolean isStripedBlockID(long id) { +return id 0; + } + + public static long convertToGroupID(long id) { +return id (~(HdfsConstants.MAX_BLOCKS_IN_GROUP - 1)); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/86bc54a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index f155375..ec07a5f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1876,7 +1876,7 @@ public class BlockManager { break; } -BlockInfoContiguous bi = blocksMap.getStoredBlock(b); +BlockInfoContiguous bi = getStoredBlock(b); if (bi == null) { if (LOG.isDebugEnabled()) { LOG.debug(BLOCK* rescanPostponedMisreplicatedBlocks: + @@ -1979,7 +1979,7 @@ public class BlockManager { continue; } - BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(iblk); + BlockInfoContiguous storedBlock = getStoredBlock(iblk); // If block does not belong to any file, we are done. if (storedBlock == null) continue; @@ -2119,7 +2119,7 @@ public class BlockManager { } // find block by blockId -BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(block); +BlockInfoContiguous storedBlock = getStoredBlock(block); if(storedBlock == null) { // If blocksMap does not contain reported block id, // the replica should be removed from the data-node. @@ -2410,7 +2410,7 @@ public class BlockManager { DatanodeDescriptor node = storageInfo.getDatanodeDescriptor(); if (block instanceof BlockInfoContiguousUnderConstruction) { //refresh our copy in case the block got completed in another thread - storedBlock = blocksMap.getStoredBlock(block); + storedBlock = getStoredBlock(block); } else { storedBlock = block; } @@ -3275,7 +3275,15 @@ public class BlockManager { } public BlockInfoContiguous getStoredBlock(Block block) { -return blocksMap.getStoredBlock(block); +BlockInfoContiguous info = null; +if (BlockIdManager.isStripedBlockID(block.getBlockId())) { + info = blocksMap.getStoredBlock( + new Block(BlockIdManager.convertToGroupID(block.getBlockId(; +} +if (info == null) { + info = blocksMap.getStoredBlock(block); +} +return info; } /** updates a block in under replication queue */
[44/52] [abbrv] hadoop git commit: HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang
HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6caf9c77 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6caf9c77 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6caf9c77 Branch: refs/heads/HDFS-7285 Commit: 6caf9c77c60321897c536101a801fcdba8e981ff Parents: 6d6fffc Author: Zhe Zhang z...@apache.org Authored: Mon Feb 9 10:27:14 2015 -0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Feb 23 11:19:51 2015 -0800 -- .../server/blockmanagement/BlockIdManager.java| 8 .../hdfs/server/blockmanagement/BlockManager.java | 18 +- 2 files changed, 21 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6caf9c77/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java index c8b9d20..e7f8a05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java @@ -211,4 +211,12 @@ public class BlockIdManager { .LAST_RESERVED_BLOCK_ID); generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP; } + + public static boolean isStripedBlockID(long id) { +return id 0; + } + + public static long convertToGroupID(long id) { +return id (~(HdfsConstants.MAX_BLOCKS_IN_GROUP - 1)); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6caf9c77/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 58a8b94..f4369b6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1876,7 +1876,7 @@ public class BlockManager { break; } -BlockInfoContiguous bi = blocksMap.getStoredBlock(b); +BlockInfoContiguous bi = getStoredBlock(b); if (bi == null) { if (LOG.isDebugEnabled()) { LOG.debug(BLOCK* rescanPostponedMisreplicatedBlocks: + @@ -1981,7 +1981,7 @@ public class BlockManager { continue; } - BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(iblk); + BlockInfoContiguous storedBlock = getStoredBlock(iblk); // If block does not belong to any file, we are done. if (storedBlock == null) continue; @@ -2123,7 +2123,7 @@ public class BlockManager { } // find block by blockId -BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(block); +BlockInfoContiguous storedBlock = getStoredBlock(block); if(storedBlock == null) { // If blocksMap does not contain reported block id, // the replica should be removed from the data-node. @@ -2414,7 +2414,7 @@ public class BlockManager { DatanodeDescriptor node = storageInfo.getDatanodeDescriptor(); if (block instanceof BlockInfoContiguousUnderConstruction) { //refresh our copy in case the block got completed in another thread - storedBlock = blocksMap.getStoredBlock(block); + storedBlock = getStoredBlock(block); } else { storedBlock = block; } @@ -3360,7 +3360,15 @@ public class BlockManager { } public BlockInfoContiguous getStoredBlock(Block block) { -return blocksMap.getStoredBlock(block); +BlockInfoContiguous info = null; +if (BlockIdManager.isStripedBlockID(block.getBlockId())) { + info = blocksMap.getStoredBlock( + new Block(BlockIdManager.convertToGroupID(block.getBlockId(; +} +if (info == null) { + info = blocksMap.getStoredBlock(block); +} +return info; } /** updates a block in under replication queue */
[3/8] hadoop git commit: HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang
HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb3132b4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb3132b4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb3132b4 Branch: refs/heads/HDFS-7285 Commit: eb3132b46f029ea6420d2787259edcdd121a9502 Parents: ae4e4d4 Author: Zhe Zhang z...@apache.org Authored: Mon Feb 9 10:27:14 2015 -0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Feb 9 17:01:21 2015 -0800 -- .../server/blockmanagement/BlockIdManager.java| 8 .../hdfs/server/blockmanagement/BlockManager.java | 18 +- 2 files changed, 21 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb3132b4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java index c8b9d20..e7f8a05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java @@ -211,4 +211,12 @@ public class BlockIdManager { .LAST_RESERVED_BLOCK_ID); generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP; } + + public static boolean isStripedBlockID(long id) { +return id 0; + } + + public static long convertToGroupID(long id) { +return id (~(HdfsConstants.MAX_BLOCKS_IN_GROUP - 1)); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb3132b4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 3fe47af..8610b79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1872,7 +1872,7 @@ public class BlockManager { break; } -BlockInfoContiguous bi = blocksMap.getStoredBlock(b); +BlockInfoContiguous bi = getStoredBlock(b); if (bi == null) { if (LOG.isDebugEnabled()) { LOG.debug(BLOCK* rescanPostponedMisreplicatedBlocks: + @@ -1977,7 +1977,7 @@ public class BlockManager { continue; } - BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(iblk); + BlockInfoContiguous storedBlock = getStoredBlock(iblk); // If block does not belong to any file, we are done. if (storedBlock == null) continue; @@ -2119,7 +2119,7 @@ public class BlockManager { } // find block by blockId -BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(block); +BlockInfoContiguous storedBlock = getStoredBlock(block); if(storedBlock == null) { // If blocksMap does not contain reported block id, // the replica should be removed from the data-node. @@ -2410,7 +2410,7 @@ public class BlockManager { DatanodeDescriptor node = storageInfo.getDatanodeDescriptor(); if (block instanceof BlockInfoContiguousUnderConstruction) { //refresh our copy in case the block got completed in another thread - storedBlock = blocksMap.getStoredBlock(block); + storedBlock = getStoredBlock(block); } else { storedBlock = block; } @@ -3356,7 +3356,15 @@ public class BlockManager { } public BlockInfoContiguous getStoredBlock(Block block) { -return blocksMap.getStoredBlock(block); +BlockInfoContiguous info = null; +if (BlockIdManager.isStripedBlockID(block.getBlockId())) { + info = blocksMap.getStoredBlock( + new Block(BlockIdManager.convertToGroupID(block.getBlockId(; +} +if (info == null) { + info = blocksMap.getStoredBlock(block); +} +return info; } /** updates a block in under replication queue */
hadoop git commit: HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang
Repository: hadoop Updated Branches: refs/heads/HDFS-EC 71a67e955 - c4362d568 HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c4362d56 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c4362d56 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c4362d56 Branch: refs/heads/HDFS-EC Commit: c4362d568855e886e2b44c73106304d769b63c96 Parents: 71a67e9 Author: Zhe Zhang z...@apache.org Authored: Mon Feb 9 10:27:14 2015 -0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Feb 9 10:27:14 2015 -0800 -- .../server/blockmanagement/BlockIdManager.java| 8 .../hdfs/server/blockmanagement/BlockManager.java | 18 +- 2 files changed, 21 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4362d56/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java index c8b9d20..e7f8a05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java @@ -211,4 +211,12 @@ public class BlockIdManager { .LAST_RESERVED_BLOCK_ID); generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP; } + + public static boolean isStripedBlockID(long id) { +return id 0; + } + + public static long convertToGroupID(long id) { +return id (~(HdfsConstants.MAX_BLOCKS_IN_GROUP - 1)); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4362d56/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 3fe47af..8610b79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1872,7 +1872,7 @@ public class BlockManager { break; } -BlockInfoContiguous bi = blocksMap.getStoredBlock(b); +BlockInfoContiguous bi = getStoredBlock(b); if (bi == null) { if (LOG.isDebugEnabled()) { LOG.debug(BLOCK* rescanPostponedMisreplicatedBlocks: + @@ -1977,7 +1977,7 @@ public class BlockManager { continue; } - BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(iblk); + BlockInfoContiguous storedBlock = getStoredBlock(iblk); // If block does not belong to any file, we are done. if (storedBlock == null) continue; @@ -2119,7 +2119,7 @@ public class BlockManager { } // find block by blockId -BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(block); +BlockInfoContiguous storedBlock = getStoredBlock(block); if(storedBlock == null) { // If blocksMap does not contain reported block id, // the replica should be removed from the data-node. @@ -2410,7 +2410,7 @@ public class BlockManager { DatanodeDescriptor node = storageInfo.getDatanodeDescriptor(); if (block instanceof BlockInfoContiguousUnderConstruction) { //refresh our copy in case the block got completed in another thread - storedBlock = blocksMap.getStoredBlock(block); + storedBlock = getStoredBlock(block); } else { storedBlock = block; } @@ -3356,7 +3356,15 @@ public class BlockManager { } public BlockInfoContiguous getStoredBlock(Block block) { -return blocksMap.getStoredBlock(block); +BlockInfoContiguous info = null; +if (BlockIdManager.isStripedBlockID(block.getBlockId())) { + info = blocksMap.getStoredBlock( + new Block(BlockIdManager.convertToGroupID(block.getBlockId(; +} +if (info == null) { + info = blocksMap.getStoredBlock(block); +} +return info; } /** updates a block in under replication queue */