HDFS-7339. Allocating and persisting block groups in NameNode. Contributed by Zhe Zhang
Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/42e26e26 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/42e26e26 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/42e26e26 Branch: refs/heads/HDFS-7285 Commit: 42e26e2624b4f733df631c6f8878e8b160b49ed6 Parents: 1d63b94 Author: Zhe Zhang <z...@apache.org> Authored: Fri Jan 30 16:16:26 2015 -0800 Committer: Zhe Zhang <z...@apache.org> Committed: Mon Mar 30 10:11:23 2015 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 + .../hadoop/hdfs/protocol/HdfsConstants.java | 4 + .../server/blockmanagement/BlockIdManager.java | 8 +- .../SequentialBlockGroupIdGenerator.java | 82 +++++++++++++++++++ .../SequentialBlockIdGenerator.java | 6 +- .../hdfs/server/namenode/FSDirectory.java | 8 +- .../hdfs/server/namenode/FSNamesystem.java | 34 +++++--- .../hadoop/hdfs/server/namenode/INodeFile.java | 11 +++ .../hdfs/server/namenode/TestAddBlockgroup.java | 84 ++++++++++++++++++++ 9 files changed, 223 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/42e26e26/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 610932a..eff457c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -221,6 +221,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT = 3; public static final String DFS_NAMENODE_REPLICATION_MIN_KEY = "dfs.namenode.replication.min"; public static final int DFS_NAMENODE_REPLICATION_MIN_DEFAULT = 1; + public static final String DFS_NAMENODE_STRIPE_MIN_KEY = "dfs.namenode.stripe.min"; + public static final int DFS_NAMENODE_STRIPE_MIN_DEFAULT = 1; public static final String DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY = "dfs.namenode.replication.pending.timeout-sec"; public static final int DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_DEFAULT = -1; public static final String DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY = "dfs.namenode.replication.max-streams"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/42e26e26/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index 54c650b..de60b6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -181,4 +181,8 @@ public class HdfsConstants { public static final byte WARM_STORAGE_POLICY_ID = 5; public static final byte EC_STORAGE_POLICY_ID = 4; public static final byte COLD_STORAGE_POLICY_ID = 2; + + public static final byte NUM_DATA_BLOCKS = 3; + public static final byte NUM_PARITY_BLOCKS = 2; + public static final byte MAX_BLOCKS_IN_GROUP = 16; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/42e26e26/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java index 1c69203..c8b9d20 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java @@ -53,10 +53,12 @@ public class BlockIdManager { * The global block ID space for this file system. */ private final SequentialBlockIdGenerator blockIdGenerator; + private final SequentialBlockGroupIdGenerator blockGroupIdGenerator; public BlockIdManager(BlockManager blockManager) { this.generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP; this.blockIdGenerator = new SequentialBlockIdGenerator(blockManager); + this.blockGroupIdGenerator = new SequentialBlockGroupIdGenerator(blockManager); } /** @@ -190,6 +192,10 @@ public class BlockIdManager { return blockIdGenerator.nextValue(); } + public long nextBlockGroupId() { + return blockGroupIdGenerator.nextValue(); + } + public boolean isGenStampInFuture(Block block) { if (isLegacyBlock(block)) { return block.getGenerationStamp() > getGenerationStampV1(); @@ -205,4 +211,4 @@ public class BlockIdManager { .LAST_RESERVED_BLOCK_ID); generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/42e26e26/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java new file mode 100644 index 0000000..e9e22ee --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.blockmanagement; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.util.SequentialNumber; + +/** + * Generate the next valid block group ID by incrementing the maximum block + * group ID allocated so far, with the first 2^10 block group IDs reserved. + * HDFS-EC introduces a hierarchical protocol to name blocks and groups: + * Contiguous: {reserved block IDs | flag | block ID} + * Striped: {reserved block IDs | flag | block group ID | index in group} + * + * Following n bits of reserved block IDs, The (n+1)th bit in an ID + * distinguishes contiguous (0) and striped (1) blocks. For a striped block, + * bits (n+2) to (64-m) represent the ID of its block group, while the last m + * bits represent its index of the group. The value m is determined by the + * maximum number of blocks in a group (MAX_BLOCKS_IN_GROUP). + */ +@InterfaceAudience.Private +public class SequentialBlockGroupIdGenerator extends SequentialNumber { + + private final BlockManager blockManager; + + SequentialBlockGroupIdGenerator(BlockManager blockManagerRef) { + super(Long.MIN_VALUE); + this.blockManager = blockManagerRef; + } + + @Override // NumberGenerator + public long nextValue() { + // Skip to next legitimate block group ID based on the naming protocol + while (super.getCurrentValue() % HdfsConstants.MAX_BLOCKS_IN_GROUP > 0) { + super.nextValue(); + } + // Make sure there's no conflict with existing random block IDs + while (hasValidBlockInRange(super.getCurrentValue())) { + super.skipTo(super.getCurrentValue() + + HdfsConstants.MAX_BLOCKS_IN_GROUP); + } + if (super.getCurrentValue() >= 0) { + BlockManager.LOG.warn("All negative block group IDs are used, " + + "growing into positive IDs, " + + "which might conflict with non-erasure coded blocks."); + } + return super.getCurrentValue(); + } + + /** + * + * @param id The starting ID of the range + * @return true if any ID in the range + * {id, id+HdfsConstants.MAX_BLOCKS_IN_GROUP} is pointed-to by a file + */ + private boolean hasValidBlockInRange(long id) { + for (int i = 0; i < HdfsConstants.MAX_BLOCKS_IN_GROUP; i++) { + Block b = new Block(id + i); + if (blockManager.getBlockCollection(b) != null) { + return true; + } + } + return false; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/42e26e26/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java index eef8857..c97de4b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.util.SequentialNumber; /** @@ -54,6 +53,11 @@ public class SequentialBlockIdGenerator extends SequentialNumber { while(isValidBlock(b)) { b.setBlockId(super.nextValue()); } + if (b.getBlockId() < 0) { + BlockManager.LOG.warn("All positive block IDs are used, " + + "wrapping to negative IDs, " + + "which might conflict with erasure coded block groups."); + } return b.getBlockId(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/42e26e26/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 7eea343..bbb6b19 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -473,10 +473,14 @@ public class FSDirectory implements Closeable { * Add a block to the file. Returns a reference to the added block. */ BlockInfoContiguous addBlock(String path, INodesInPath inodesInPath, - Block block, DatanodeStorageInfo[] targets) throws IOException { + Block block, DatanodeStorageInfo[] targets, + boolean isStriped) throws IOException { writeLock(); try { final INodeFile fileINode = inodesInPath.getLastINode().asFile(); + short numLocations = isStriped ? + HdfsConstants.NUM_DATA_BLOCKS + HdfsConstants.NUM_PARITY_BLOCKS : + fileINode.getFileReplication(); Preconditions.checkState(fileINode.isUnderConstruction()); // check quota limits and updated space consumed @@ -487,7 +491,7 @@ public class FSDirectory implements Closeable { BlockInfoContiguousUnderConstruction blockInfo = new BlockInfoContiguousUnderConstruction( block, - fileINode.getFileReplication(), + numLocations, BlockUCState.UNDER_CONSTRUCTION, targets); getBlockManager().addBlockCollection(blockInfo, fileINode); http://git-wip-us.apache.org/repos/asf/hadoop/blob/42e26e26/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 0e0f484..c4768ee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2065,7 +2065,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, BlockInfoContiguous oldBlock = file.getLastBlock(); boolean shouldCopyOnTruncate = shouldCopyOnTruncate(file, oldBlock); if(newBlock == null) { - newBlock = (shouldCopyOnTruncate) ? createNewBlock() : + newBlock = (shouldCopyOnTruncate) ? createNewBlock(file.isStriped()) : new Block(oldBlock.getBlockId(), oldBlock.getNumBytes(), nextGenerationStamp(blockIdManager.isLegacyBlock(oldBlock))); } @@ -3010,8 +3010,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, ExtendedBlock previous, Set<Node> excludedNodes, List<String> favoredNodes) throws IOException { final long blockSize; - final int replication; + final short numTargets; final byte storagePolicyID; + final boolean isStriped; Node clientNode = null; String clientMachine = null; @@ -3049,7 +3050,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, .getClientMachine(); clientNode = blockManager.getDatanodeManager().getDatanodeByHost( clientMachine); - replication = pendingFile.getFileReplication(); + // TODO: make block group size configurable (HDFS-7337) + isStriped = pendingFile.isStriped(); + numTargets = isStriped ? + HdfsConstants.NUM_DATA_BLOCKS + HdfsConstants.NUM_PARITY_BLOCKS : + pendingFile.getFileReplication(); storagePolicyID = pendingFile.getStoragePolicyID(); } finally { readUnlock(); @@ -3061,7 +3066,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, // choose targets for the new block to be allocated. final DatanodeStorageInfo targets[] = getBlockManager().chooseTarget4NewBlock( - src, replication, clientNode, excludedNodes, blockSize, favoredNodes, + src, numTargets, clientNode, excludedNodes, blockSize, favoredNodes, storagePolicyID); // Part II. @@ -3100,9 +3105,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, ExtendedBlock.getLocalBlock(previous)); // allocate new block, record block locations in INode. - newBlock = createNewBlock(); + newBlock = createNewBlock(isStriped); INodesInPath inodesInPath = INodesInPath.fromINode(pendingFile); - saveAllocatedBlock(src, inodesInPath, newBlock, targets); + saveAllocatedBlock(src, inodesInPath, newBlock, targets, isStriped); persistNewBlock(src, pendingFile); offset = pendingFile.computeFileSize(); @@ -3523,13 +3528,15 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, * The last INode is the INode for {@code src} file. * @param newBlock newly allocated block to be save * @param targets target datanodes where replicas of the new block is placed + * @param isStriped is the file under striping or contigunous layout? * @throws QuotaExceededException If addition of block exceeds space quota */ BlockInfoContiguous saveAllocatedBlock(String src, INodesInPath inodesInPath, - Block newBlock, DatanodeStorageInfo[] targets) + Block newBlock, DatanodeStorageInfo[] targets, boolean isStriped) throws IOException { assert hasWriteLock(); - BlockInfoContiguous b = dir.addBlock(src, inodesInPath, newBlock, targets); + BlockInfoContiguous b = dir.addBlock(src, inodesInPath, newBlock, targets, + isStriped); NameNode.stateChangeLog.info("BLOCK* allocate " + b + " for " + src); DatanodeStorageInfo.incrementBlocksScheduled(targets); return b; @@ -3537,10 +3544,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, /** * Create new block with a unique block id and a new generation stamp. + * @param isStriped is the file under striping or contiguous layout? */ - Block createNewBlock() throws IOException { + Block createNewBlock(boolean isStriped) throws IOException { assert hasWriteLock(); - Block b = new Block(nextBlockId(), 0, 0); + Block b = new Block(nextBlockId(isStriped), 0, 0); // Increment the generation stamp for every new block. b.setGenerationStamp(nextGenerationStamp(false)); return b; @@ -6146,11 +6154,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, /** * Increments, logs and then returns the block ID + * @param isStriped is the file under striping or contiguous layout? */ - private long nextBlockId() throws IOException { + private long nextBlockId(boolean isStriped) throws IOException { assert hasWriteLock(); checkNameNodeSafeMode("Cannot get next block ID"); - final long blockId = blockIdManager.nextBlockId(); + final long blockId = isStriped ? + blockIdManager.nextBlockGroupId() : blockIdManager.nextBlockId(); getEditLog().logAllocateBlockId(blockId); // NB: callers sync the log return blockId; http://git-wip-us.apache.org/repos/asf/hadoop/blob/42e26e26/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index a6f07f9..1858e0a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -34,12 +34,14 @@ import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiff; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList; @@ -918,4 +920,13 @@ public class INodeFile extends INodeWithAdditionalFields return snapshotBlocks != null && Arrays.asList(snapshotBlocks).contains(block); } + + @VisibleForTesting + /** + * @return true if the file is in the striping layout. + */ + // TODO: move erasure coding policy to file XAttr (HDFS-7337) + public boolean isStriped() { + return getStoragePolicyID() == HdfsConstants.EC_STORAGE_POLICY_ID; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/42e26e26/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockgroup.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockgroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockgroup.java new file mode 100644 index 0000000..95133ce --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockgroup.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +public class TestAddBlockgroup { + + public static final Log LOG = LogFactory.getLog(TestAddBlockgroup.class); + + private final short GROUP_SIZE = HdfsConstants.NUM_DATA_BLOCKS + + HdfsConstants.NUM_PARITY_BLOCKS; + private final short NUM_DATANODES = GROUP_SIZE; + + private static final int BLOCKSIZE = 1024; + private static final short REPLICATION = 3; + + private MiniDFSCluster cluster; + private Configuration conf; + + @Before + public void setup() throws IOException { + conf = new Configuration(); + conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES) + .build(); + cluster.waitActive(); + cluster.getFileSystem().setStoragePolicy(new Path("/"), + HdfsConstants.EC_STORAGE_POLICY_NAME); + } + + @After + public void tearDown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + public void testAddBlockGroup() throws Exception { + DistributedFileSystem fs = cluster.getFileSystem(); + FSDirectory fsdir = cluster.getNamesystem().getFSDirectory(); + + final Path file1 = new Path("/file1"); + DFSTestUtil.createFile(fs, file1, BLOCKSIZE * 2, REPLICATION, 0L); + INodeFile file1Node = fsdir.getINode4Write(file1.toString()).asFile(); + BlockInfo[] file1Blocks = file1Node.getBlocks(); + assertEquals(2, file1Blocks.length); + assertEquals(GROUP_SIZE, file1Blocks[0].numNodes()); + assertEquals(HdfsConstants.MAX_BLOCKS_IN_GROUP, + file1Blocks[1].getBlockId() - file1Blocks[0].getBlockId()); + } +}