Repository: hadoop Updated Branches: refs/heads/HDFS-7285 c4fcea80f -> 35bb92fe2
HDFS-8316. Erasure coding: refactor EC constants to be consistent with HDFS-8249. Contributed by Zhe Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/35bb92fe Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/35bb92fe Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/35bb92fe Branch: refs/heads/HDFS-7285 Commit: 35bb92fe2afda92a5ba41d48d41aac6723a141b3 Parents: c4fcea8 Author: Jing Zhao <ji...@apache.org> Authored: Mon May 4 11:24:35 2015 -0700 Committer: Jing Zhao <ji...@apache.org> Committed: Mon May 4 11:24:35 2015 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/protocol/HdfsConstants.java | 11 +++++++++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +++ .../org/apache/hadoop/hdfs/DFSStripedOutputStream.java | 2 +- .../hdfs/server/blockmanagement/BlockIdManager.java | 4 ++-- .../blockmanagement/SequentialBlockGroupIdGenerator.java | 4 ++-- .../hadoop/hdfs/server/common/HdfsServerConstants.java | 5 ++++- .../hdfs/server/namenode/TestAddStripedBlocks.java | 4 ++-- .../hdfs/server/namenode/TestStripedINodeFile.java | 6 +++--- 8 files changed, 28 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/35bb92fe/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index 58c7ea1..32ca81c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -75,6 +75,17 @@ public final class HdfsConstants { public static final String CLIENT_NAMENODE_PROTOCOL_NAME = "org.apache.hadoop.hdfs.protocol.ClientProtocol"; + /* + * These values correspond to the values used by the system default erasure + * coding schema. + * TODO: to be removed once all places use schema. + */ + + public static final byte NUM_DATA_BLOCKS = 6; + public static final byte NUM_PARITY_BLOCKS = 3; + // The chunk size for striped block which is used by erasure coding + public static final int BLOCK_STRIPED_CELL_SIZE = 256 * 1024; + // SafeMode actions public enum SafeModeAction { SAFEMODE_LEAVE, SAFEMODE_ENTER, SAFEMODE_GET http://git-wip-us.apache.org/repos/asf/hadoop/blob/35bb92fe/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 145494f..e30b2ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -158,3 +158,6 @@ HDFS-7949. WebImageViewer need support file size calculation with striped blocks. (Rakesh R via Zhe Zhang) + + HDFS-8316. Erasure coding: refactor EC constants to be consistent with HDFS-8249. + (Zhe Zhang via jing9) http://git-wip-us.apache.org/repos/asf/hadoop/blob/35bb92fe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index 5e2a534..71cdbb9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -419,7 +419,7 @@ public class DFSStripedOutputStream extends DFSOutputStream { @Override protected synchronized void closeImpl() throws IOException { if (isClosed()) { - getLeadingStreamer().getLastException().check(); + getLeadingStreamer().getLastException().check(true); return; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/35bb92fe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java index fc82d5d..746802c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java @@ -241,11 +241,11 @@ public class BlockIdManager { * data/parity block id in the same striped block group. */ public static long convertToStripedID(long id) { - return id & (~HdfsConstants.BLOCK_GROUP_INDEX_MASK); + return id & (~HdfsServerConstants.BLOCK_GROUP_INDEX_MASK); } public static int getBlockIndex(Block reportedBlock) { return (int) (reportedBlock.getBlockId() & - HdfsConstants.BLOCK_GROUP_INDEX_MASK); + HdfsServerConstants.BLOCK_GROUP_INDEX_MASK); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/35bb92fe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java index de8e379..479ee4c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java @@ -21,8 +21,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.util.SequentialNumber; -import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_GROUP_INDEX_MASK; -import static org.apache.hadoop.hdfs.protocol.HdfsConstants.MAX_BLOCKS_IN_GROUP; +import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BLOCK_GROUP_INDEX_MASK; +import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.MAX_BLOCKS_IN_GROUP; /** * Generate the next valid block group ID by incrementing the maximum block http://git-wip-us.apache.org/repos/asf/hadoop/blob/35bb92fe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java index ed3e84f..a0ae43b0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java @@ -388,6 +388,9 @@ public interface HdfsServerConstants { "raw.hdfs.crypto.file.encryption.info"; String SECURITY_XATTR_UNREADABLE_BY_SUPERUSER = "security.hdfs.unreadable.by.superuser"; - public static final String XATTR_ERASURECODING_ZONE = + String XATTR_ERASURECODING_ZONE = "raw.hdfs.erasurecoding.zone"; + + long BLOCK_GROUP_INDEX_MASK = 15; + byte MAX_BLOCKS_IN_GROUP = 16; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/35bb92fe/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java index 297db14..ba76360 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java @@ -102,7 +102,7 @@ public class TestAddStripedBlocks { DFSTestUtil.writeFile(dfs, testPath, "hello again"); lb = dfs.getClient().getLocatedBlocks(testPath.toString(), 0); final long secondId = lb.get(0).getBlock().getBlockId(); - Assert.assertEquals(firstId + HdfsConstants.MAX_BLOCKS_IN_GROUP, secondId); + Assert.assertEquals(firstId + HdfsServerConstants.MAX_BLOCKS_IN_GROUP, secondId); } @Test (timeout=60000) @@ -159,7 +159,7 @@ public class TestAddStripedBlocks { Assert.assertEquals(HdfsConstants.NUM_PARITY_BLOCKS, block.getParityBlockNum()); Assert.assertEquals(0, - block.getBlockId() & HdfsConstants.BLOCK_GROUP_INDEX_MASK); + block.getBlockId() & HdfsServerConstants.BLOCK_GROUP_INDEX_MASK); final BlockInfoStripedUnderConstruction blockUC = (BlockInfoStripedUnderConstruction) block; http://git-wip-us.apache.org/repos/asf/hadoop/blob/35bb92fe/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java index 4a6d6cc..f97943b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java @@ -28,10 +28,10 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.protocol.HdfsConstantsClient; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStripedUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.io.erasurecode.ECSchema; import org.junit.Test; @@ -46,8 +46,8 @@ public class TestStripedINodeFile { "userName", null, FsPermission.getDefault()); private static INodeFile createStripedINodeFile() { - return new INodeFile(HdfsConstantsClient.GRANDFATHER_INODE_ID, null, perm, 0L, 0L, - null, (short)0, 1024L, HdfsConstants.COLD_STORAGE_POLICY_ID); + return new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID, null, perm, 0L, 0L, + null, (short)0, 1024L, HdfsServerConstants.COLD_STORAGE_POLICY_ID); } @Test