[30/50] [abbrv] hadoop git commit: HDFS-7716. Erasure Coding: extend BlockInfo to handle EC info. Contributed by Jing Zhao.
HDFS-7716. Erasure Coding: extend BlockInfo to handle EC info. Contributed by Jing Zhao. Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/249ca621 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/249ca621 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/249ca621 Branch: refs/heads/HDFS-7285 Commit: 249ca621cfbc8c23d2da36eb9b7336349aef69bd Parents: 97ce9b9 Author: Jing Zhao ji...@apache.org Authored: Tue Feb 10 17:54:10 2015 -0800 Committer: Zhe Zhang z...@apache.org Committed: Tue Mar 24 11:16:32 2015 -0700 -- .../hadoop/hdfs/protocol/HdfsConstants.java | 1 + .../server/blockmanagement/BlockCollection.java | 13 +- .../server/blockmanagement/BlockIdManager.java | 7 +- .../hdfs/server/blockmanagement/BlockInfo.java | 339 + .../blockmanagement/BlockInfoContiguous.java| 363 +++ .../BlockInfoContiguousUnderConstruction.java | 137 +-- .../blockmanagement/BlockInfoStriped.java | 179 + .../server/blockmanagement/BlockManager.java| 188 +- .../hdfs/server/blockmanagement/BlocksMap.java | 46 +-- .../CacheReplicationMonitor.java| 10 +- .../blockmanagement/DatanodeDescriptor.java | 22 +- .../blockmanagement/DatanodeStorageInfo.java| 38 +- .../ReplicaUnderConstruction.java | 119 ++ .../hdfs/server/namenode/FSDirectory.java | 4 +- .../hdfs/server/namenode/FSNamesystem.java | 24 +- .../hdfs/server/namenode/NamenodeFsck.java | 3 +- .../snapshot/FSImageFormatPBSnapshot.java | 4 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 4 +- .../server/blockmanagement/TestBlockInfo.java | 6 +- .../blockmanagement/TestBlockInfoStriped.java | 219 +++ .../blockmanagement/TestBlockManager.java | 4 +- .../blockmanagement/TestReplicationPolicy.java | 2 +- 22 files changed, 1125 insertions(+), 607 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/249ca621/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index de60b6e..245b630 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -184,5 +184,6 @@ public class HdfsConstants { public static final byte NUM_DATA_BLOCKS = 3; public static final byte NUM_PARITY_BLOCKS = 2; + public static final long BLOCK_GROUP_INDEX_MASK = 15; public static final byte MAX_BLOCKS_IN_GROUP = 16; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/249ca621/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java index 1547611..974cac3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java @@ -39,12 +39,12 @@ public interface BlockCollection { public ContentSummary computeContentSummary(); /** - * @return the number of blocks + * @return the number of blocks or block groups */ public int numBlocks(); /** - * Get the blocks. + * Get the blocks or block groups. */ public BlockInfoContiguous[] getBlocks(); @@ -55,8 +55,8 @@ public interface BlockCollection { public long getPreferredBlockSize(); /** - * Get block replication for the collection - * @return block replication value + * Get block replication for the collection. + * @return block replication value. Return 0 if the file is erasure coded. */ public short getBlockReplication(); @@ -71,7 +71,7 @@ public interface BlockCollection { public String getName(); /** - * Set the block at the given index. + * Set the block/block-group at the given index. */ public void setBlock(int index, BlockInfoContiguous blk); @@ -79,7 +79,8 @@ public interface
[30/50] [abbrv] hadoop git commit: HDFS-7716. Erasure Coding: extend BlockInfo to handle EC info. Contributed by Jing Zhao.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d515e7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index c4612a3..3a5e66e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -24,6 +24,7 @@ import java.util.List; import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State; @@ -80,10 +81,10 @@ public class DatanodeStorageInfo { /** * Iterates over the list of blocks belonging to the data-node. */ - class BlockIterator implements IteratorBlockInfoContiguous { -private BlockInfoContiguous current; + class BlockIterator implements IteratorBlockInfo { +private BlockInfo current; -BlockIterator(BlockInfoContiguous head) { +BlockIterator(BlockInfo head) { this.current = head; } @@ -91,8 +92,8 @@ public class DatanodeStorageInfo { return current != null; } -public BlockInfoContiguous next() { - BlockInfoContiguous res = current; +public BlockInfo next() { + BlockInfo res = current; current = current.getNext(current.findStorageInfo(DatanodeStorageInfo.this)); return res; } @@ -112,7 +113,7 @@ public class DatanodeStorageInfo { private volatile long remaining; private long blockPoolUsed; - private volatile BlockInfoContiguous blockList = null; + private volatile BlockInfo blockList = null; private int numBlocks = 0; /** The number of block reports received */ @@ -215,7 +216,7 @@ public class DatanodeStorageInfo { return blockPoolUsed; } - public AddBlockResult addBlock(BlockInfoContiguous b) { + public AddBlockResult addBlock(BlockInfo b, Block reportedBlock) { // First check whether the block belongs to a different storage // on the same DN. AddBlockResult result = AddBlockResult.ADDED; @@ -234,13 +235,21 @@ public class DatanodeStorageInfo { } // add to the head of the data-node list -b.addStorage(this); +b.addStorage(this, reportedBlock); +insertToList(b); +return result; + } + + AddBlockResult addBlock(BlockInfoContiguous b) { +return addBlock(b, b); + } + + public void insertToList(BlockInfo b) { blockList = b.listInsert(blockList, this); numBlocks++; -return result; } - public boolean removeBlock(BlockInfoContiguous b) { + public boolean removeBlock(BlockInfo b) { blockList = b.listRemove(blockList, this); if (b.removeStorage(this)) { numBlocks--; @@ -254,16 +263,15 @@ public class DatanodeStorageInfo { return numBlocks; } - IteratorBlockInfoContiguous getBlockIterator() { + IteratorBlockInfo getBlockIterator() { return new BlockIterator(blockList); - } /** * Move block to the head of the list of blocks belonging to the data-node. * @return the index of the head of the blockList */ - int moveBlockToHead(BlockInfoContiguous b, int curIndex, int headIndex) { + int moveBlockToHead(BlockInfo b, int curIndex, int headIndex) { blockList = b.moveBlockToHead(blockList, this, curIndex, headIndex); return curIndex; } @@ -273,7 +281,7 @@ public class DatanodeStorageInfo { * @return the head of the blockList */ @VisibleForTesting - BlockInfoContiguous getBlockListHeadForTesting(){ + BlockInfo getBlockListHeadForTesting(){ return blockList; } @@ -360,6 +368,6 @@ public class DatanodeStorageInfo { } static enum AddBlockResult { -ADDED, REPLACED, ALREADY_EXIST; +ADDED, REPLACED, ALREADY_EXIST } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d515e7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicaUnderConstruction.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicaUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicaUnderConstruction.java new file mode 100644 index 000..f4600cb7 --- /dev/null +++