[14/50] hadoop git commit: HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao.
HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/98ebb4ed Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/98ebb4ed Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/98ebb4ed Branch: refs/heads/HDFS-7285 Commit: 98ebb4eda1916e1db5057796e6fa075781366214 Parents: b4a33f2 Author: Zhe Zhang z...@apache.org Authored: Fri Apr 24 09:30:38 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon May 18 10:01:48 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 ++ .../SequentialBlockGroupIdGenerator.java| 39 +++--- .../SequentialBlockIdGenerator.java | 2 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 57 +++- .../server/namenode/TestAddStripedBlocks.java | 21 5 files changed, 77 insertions(+), 45 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/98ebb4ed/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 9357e23..cf41a9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -128,3 +128,6 @@ HDFS-8223. Should calculate checksum for parity blocks in DFSStripedOutputStream. (Yi Liu via jing9) + +HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause +block id conflicts (Jing Zhao via Zhe Zhang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/98ebb4ed/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java index e9e22ee..de8e379 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java @@ -19,9 +19,11 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.util.SequentialNumber; +import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_GROUP_INDEX_MASK; +import static org.apache.hadoop.hdfs.protocol.HdfsConstants.MAX_BLOCKS_IN_GROUP; + /** * Generate the next valid block group ID by incrementing the maximum block * group ID allocated so far, with the first 2^10 block group IDs reserved. @@ -34,6 +36,9 @@ import org.apache.hadoop.util.SequentialNumber; * bits (n+2) to (64-m) represent the ID of its block group, while the last m * bits represent its index of the group. The value m is determined by the * maximum number of blocks in a group (MAX_BLOCKS_IN_GROUP). + * + * Note that the {@link #nextValue()} methods requires external lock to + * guarantee IDs have no conflicts. */ @InterfaceAudience.Private public class SequentialBlockGroupIdGenerator extends SequentialNumber { @@ -47,32 +52,30 @@ public class SequentialBlockGroupIdGenerator extends SequentialNumber { @Override // NumberGenerator public long nextValue() { -// Skip to next legitimate block group ID based on the naming protocol -while (super.getCurrentValue() % HdfsConstants.MAX_BLOCKS_IN_GROUP 0) { - super.nextValue(); -} +skipTo((getCurrentValue() ~BLOCK_GROUP_INDEX_MASK) + MAX_BLOCKS_IN_GROUP); // Make sure there's no conflict with existing random block IDs -while (hasValidBlockInRange(super.getCurrentValue())) { - super.skipTo(super.getCurrentValue() + - HdfsConstants.MAX_BLOCKS_IN_GROUP); +final Block b = new Block(getCurrentValue()); +while (hasValidBlockInRange(b)) { + skipTo(getCurrentValue() + MAX_BLOCKS_IN_GROUP); + b.setBlockId(getCurrentValue()); } -if (super.getCurrentValue() = 0) { - BlockManager.LOG.warn(All negative block group IDs are used, + - growing into positive IDs, + - which might conflict with non-erasure coded blocks.); +if (b.getBlockId() = 0) { + throw new IllegalStateException(All
[11/50] hadoop git commit: HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao.
HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/851b1145 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/851b1145 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/851b1145 Branch: refs/heads/HDFS-7285 Commit: 851b1145ba9c7ba2b24787d26152f373381ee565 Parents: cb8dd8a Author: Zhe Zhang z...@apache.org Authored: Fri Apr 24 09:30:38 2015 -0700 Committer: Jing Zhao ji...@apache.org Committed: Mon May 18 22:11:06 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 ++ .../SequentialBlockGroupIdGenerator.java| 39 +++--- .../SequentialBlockIdGenerator.java | 2 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 57 +++- .../server/namenode/TestAddStripedBlocks.java | 21 5 files changed, 77 insertions(+), 45 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/851b1145/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 9357e23..cf41a9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -128,3 +128,6 @@ HDFS-8223. Should calculate checksum for parity blocks in DFSStripedOutputStream. (Yi Liu via jing9) + +HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause +block id conflicts (Jing Zhao via Zhe Zhang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/851b1145/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java index e9e22ee..de8e379 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java @@ -19,9 +19,11 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.util.SequentialNumber; +import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_GROUP_INDEX_MASK; +import static org.apache.hadoop.hdfs.protocol.HdfsConstants.MAX_BLOCKS_IN_GROUP; + /** * Generate the next valid block group ID by incrementing the maximum block * group ID allocated so far, with the first 2^10 block group IDs reserved. @@ -34,6 +36,9 @@ import org.apache.hadoop.util.SequentialNumber; * bits (n+2) to (64-m) represent the ID of its block group, while the last m * bits represent its index of the group. The value m is determined by the * maximum number of blocks in a group (MAX_BLOCKS_IN_GROUP). + * + * Note that the {@link #nextValue()} methods requires external lock to + * guarantee IDs have no conflicts. */ @InterfaceAudience.Private public class SequentialBlockGroupIdGenerator extends SequentialNumber { @@ -47,32 +52,30 @@ public class SequentialBlockGroupIdGenerator extends SequentialNumber { @Override // NumberGenerator public long nextValue() { -// Skip to next legitimate block group ID based on the naming protocol -while (super.getCurrentValue() % HdfsConstants.MAX_BLOCKS_IN_GROUP 0) { - super.nextValue(); -} +skipTo((getCurrentValue() ~BLOCK_GROUP_INDEX_MASK) + MAX_BLOCKS_IN_GROUP); // Make sure there's no conflict with existing random block IDs -while (hasValidBlockInRange(super.getCurrentValue())) { - super.skipTo(super.getCurrentValue() + - HdfsConstants.MAX_BLOCKS_IN_GROUP); +final Block b = new Block(getCurrentValue()); +while (hasValidBlockInRange(b)) { + skipTo(getCurrentValue() + MAX_BLOCKS_IN_GROUP); + b.setBlockId(getCurrentValue()); } -if (super.getCurrentValue() = 0) { - BlockManager.LOG.warn(All negative block group IDs are used, + - growing into positive IDs, + - which might conflict with non-erasure coded blocks.); +if (b.getBlockId() = 0) { + throw new IllegalStateException(All
[16/50] hadoop git commit: HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao.
HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f5c6ec66 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f5c6ec66 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f5c6ec66 Branch: refs/heads/HDFS-7285 Commit: f5c6ec66b6cb19d287d69eae3aac964f79ca552b Parents: c9627f8 Author: Zhe Zhang z...@apache.org Authored: Fri Apr 24 09:30:38 2015 -0700 Committer: Jing Zhao ji...@apache.org Committed: Sat May 16 15:15:23 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 ++ .../SequentialBlockGroupIdGenerator.java| 39 +++--- .../SequentialBlockIdGenerator.java | 2 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 57 +++- .../server/namenode/TestAddStripedBlocks.java | 21 5 files changed, 77 insertions(+), 45 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5c6ec66/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 9357e23..cf41a9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -128,3 +128,6 @@ HDFS-8223. Should calculate checksum for parity blocks in DFSStripedOutputStream. (Yi Liu via jing9) + +HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause +block id conflicts (Jing Zhao via Zhe Zhang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5c6ec66/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java index e9e22ee..de8e379 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java @@ -19,9 +19,11 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.util.SequentialNumber; +import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_GROUP_INDEX_MASK; +import static org.apache.hadoop.hdfs.protocol.HdfsConstants.MAX_BLOCKS_IN_GROUP; + /** * Generate the next valid block group ID by incrementing the maximum block * group ID allocated so far, with the first 2^10 block group IDs reserved. @@ -34,6 +36,9 @@ import org.apache.hadoop.util.SequentialNumber; * bits (n+2) to (64-m) represent the ID of its block group, while the last m * bits represent its index of the group. The value m is determined by the * maximum number of blocks in a group (MAX_BLOCKS_IN_GROUP). + * + * Note that the {@link #nextValue()} methods requires external lock to + * guarantee IDs have no conflicts. */ @InterfaceAudience.Private public class SequentialBlockGroupIdGenerator extends SequentialNumber { @@ -47,32 +52,30 @@ public class SequentialBlockGroupIdGenerator extends SequentialNumber { @Override // NumberGenerator public long nextValue() { -// Skip to next legitimate block group ID based on the naming protocol -while (super.getCurrentValue() % HdfsConstants.MAX_BLOCKS_IN_GROUP 0) { - super.nextValue(); -} +skipTo((getCurrentValue() ~BLOCK_GROUP_INDEX_MASK) + MAX_BLOCKS_IN_GROUP); // Make sure there's no conflict with existing random block IDs -while (hasValidBlockInRange(super.getCurrentValue())) { - super.skipTo(super.getCurrentValue() + - HdfsConstants.MAX_BLOCKS_IN_GROUP); +final Block b = new Block(getCurrentValue()); +while (hasValidBlockInRange(b)) { + skipTo(getCurrentValue() + MAX_BLOCKS_IN_GROUP); + b.setBlockId(getCurrentValue()); } -if (super.getCurrentValue() = 0) { - BlockManager.LOG.warn(All negative block group IDs are used, + - growing into positive IDs, + - which might conflict with non-erasure coded blocks.); +if (b.getBlockId() = 0) { + throw new IllegalStateException(All
[25/50] hadoop git commit: HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao.
HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2baca72 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2baca72 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2baca72 Branch: refs/heads/HDFS-7285 Commit: f2baca72c5c274f97451aec20ded5d4260c4e4c7 Parents: 3627397 Author: Zhe Zhang z...@apache.org Authored: Fri Apr 24 09:30:38 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon May 11 11:36:20 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 ++ .../SequentialBlockGroupIdGenerator.java| 39 +++--- .../SequentialBlockIdGenerator.java | 2 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 57 +++- .../server/namenode/TestAddStripedBlocks.java | 21 5 files changed, 77 insertions(+), 45 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2baca72/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 9357e23..cf41a9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -128,3 +128,6 @@ HDFS-8223. Should calculate checksum for parity blocks in DFSStripedOutputStream. (Yi Liu via jing9) + +HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause +block id conflicts (Jing Zhao via Zhe Zhang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2baca72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java index e9e22ee..de8e379 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java @@ -19,9 +19,11 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.util.SequentialNumber; +import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_GROUP_INDEX_MASK; +import static org.apache.hadoop.hdfs.protocol.HdfsConstants.MAX_BLOCKS_IN_GROUP; + /** * Generate the next valid block group ID by incrementing the maximum block * group ID allocated so far, with the first 2^10 block group IDs reserved. @@ -34,6 +36,9 @@ import org.apache.hadoop.util.SequentialNumber; * bits (n+2) to (64-m) represent the ID of its block group, while the last m * bits represent its index of the group. The value m is determined by the * maximum number of blocks in a group (MAX_BLOCKS_IN_GROUP). + * + * Note that the {@link #nextValue()} methods requires external lock to + * guarantee IDs have no conflicts. */ @InterfaceAudience.Private public class SequentialBlockGroupIdGenerator extends SequentialNumber { @@ -47,32 +52,30 @@ public class SequentialBlockGroupIdGenerator extends SequentialNumber { @Override // NumberGenerator public long nextValue() { -// Skip to next legitimate block group ID based on the naming protocol -while (super.getCurrentValue() % HdfsConstants.MAX_BLOCKS_IN_GROUP 0) { - super.nextValue(); -} +skipTo((getCurrentValue() ~BLOCK_GROUP_INDEX_MASK) + MAX_BLOCKS_IN_GROUP); // Make sure there's no conflict with existing random block IDs -while (hasValidBlockInRange(super.getCurrentValue())) { - super.skipTo(super.getCurrentValue() + - HdfsConstants.MAX_BLOCKS_IN_GROUP); +final Block b = new Block(getCurrentValue()); +while (hasValidBlockInRange(b)) { + skipTo(getCurrentValue() + MAX_BLOCKS_IN_GROUP); + b.setBlockId(getCurrentValue()); } -if (super.getCurrentValue() = 0) { - BlockManager.LOG.warn(All negative block group IDs are used, + - growing into positive IDs, + - which might conflict with non-erasure coded blocks.); +if (b.getBlockId() = 0) { + throw new IllegalStateException(All
[41/50] hadoop git commit: HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao.
HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/395e29bb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/395e29bb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/395e29bb Branch: refs/heads/HDFS-7285 Commit: 395e29bb0bf4f46722a0e5cea0bffba64052edde Parents: 254759d Author: Zhe Zhang z...@apache.org Authored: Fri Apr 24 09:30:38 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon May 4 10:13:30 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 ++ .../SequentialBlockGroupIdGenerator.java| 39 +++--- .../SequentialBlockIdGenerator.java | 2 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 57 +++- .../server/namenode/TestAddStripedBlocks.java | 21 5 files changed, 77 insertions(+), 45 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/395e29bb/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 9357e23..cf41a9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -128,3 +128,6 @@ HDFS-8223. Should calculate checksum for parity blocks in DFSStripedOutputStream. (Yi Liu via jing9) + +HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause +block id conflicts (Jing Zhao via Zhe Zhang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/395e29bb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java index e9e22ee..de8e379 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java @@ -19,9 +19,11 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.util.SequentialNumber; +import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_GROUP_INDEX_MASK; +import static org.apache.hadoop.hdfs.protocol.HdfsConstants.MAX_BLOCKS_IN_GROUP; + /** * Generate the next valid block group ID by incrementing the maximum block * group ID allocated so far, with the first 2^10 block group IDs reserved. @@ -34,6 +36,9 @@ import org.apache.hadoop.util.SequentialNumber; * bits (n+2) to (64-m) represent the ID of its block group, while the last m * bits represent its index of the group. The value m is determined by the * maximum number of blocks in a group (MAX_BLOCKS_IN_GROUP). + * + * Note that the {@link #nextValue()} methods requires external lock to + * guarantee IDs have no conflicts. */ @InterfaceAudience.Private public class SequentialBlockGroupIdGenerator extends SequentialNumber { @@ -47,32 +52,30 @@ public class SequentialBlockGroupIdGenerator extends SequentialNumber { @Override // NumberGenerator public long nextValue() { -// Skip to next legitimate block group ID based on the naming protocol -while (super.getCurrentValue() % HdfsConstants.MAX_BLOCKS_IN_GROUP 0) { - super.nextValue(); -} +skipTo((getCurrentValue() ~BLOCK_GROUP_INDEX_MASK) + MAX_BLOCKS_IN_GROUP); // Make sure there's no conflict with existing random block IDs -while (hasValidBlockInRange(super.getCurrentValue())) { - super.skipTo(super.getCurrentValue() + - HdfsConstants.MAX_BLOCKS_IN_GROUP); +final Block b = new Block(getCurrentValue()); +while (hasValidBlockInRange(b)) { + skipTo(getCurrentValue() + MAX_BLOCKS_IN_GROUP); + b.setBlockId(getCurrentValue()); } -if (super.getCurrentValue() = 0) { - BlockManager.LOG.warn(All negative block group IDs are used, + - growing into positive IDs, + - which might conflict with non-erasure coded blocks.); +if (b.getBlockId() = 0) { + throw new IllegalStateException(All
[41/50] hadoop git commit: HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao.
HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1e7dfa0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1e7dfa0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1e7dfa0 Branch: refs/heads/HDFS-7285 Commit: d1e7dfa0e0e9bf76f4925936f27b1768339cb5dd Parents: f24c2c0 Author: Zhe Zhang z...@apache.org Authored: Fri Apr 24 09:30:38 2015 -0700 Committer: Jing Zhao ji...@apache.org Committed: Wed Apr 29 11:16:56 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 ++ .../SequentialBlockGroupIdGenerator.java| 39 +++--- .../SequentialBlockIdGenerator.java | 2 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 57 +++- .../server/namenode/TestAddStripedBlocks.java | 21 5 files changed, 77 insertions(+), 45 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1e7dfa0/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 9357e23..cf41a9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -128,3 +128,6 @@ HDFS-8223. Should calculate checksum for parity blocks in DFSStripedOutputStream. (Yi Liu via jing9) + +HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause +block id conflicts (Jing Zhao via Zhe Zhang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1e7dfa0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java index e9e22ee..de8e379 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java @@ -19,9 +19,11 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.util.SequentialNumber; +import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_GROUP_INDEX_MASK; +import static org.apache.hadoop.hdfs.protocol.HdfsConstants.MAX_BLOCKS_IN_GROUP; + /** * Generate the next valid block group ID by incrementing the maximum block * group ID allocated so far, with the first 2^10 block group IDs reserved. @@ -34,6 +36,9 @@ import org.apache.hadoop.util.SequentialNumber; * bits (n+2) to (64-m) represent the ID of its block group, while the last m * bits represent its index of the group. The value m is determined by the * maximum number of blocks in a group (MAX_BLOCKS_IN_GROUP). + * + * Note that the {@link #nextValue()} methods requires external lock to + * guarantee IDs have no conflicts. */ @InterfaceAudience.Private public class SequentialBlockGroupIdGenerator extends SequentialNumber { @@ -47,32 +52,30 @@ public class SequentialBlockGroupIdGenerator extends SequentialNumber { @Override // NumberGenerator public long nextValue() { -// Skip to next legitimate block group ID based on the naming protocol -while (super.getCurrentValue() % HdfsConstants.MAX_BLOCKS_IN_GROUP 0) { - super.nextValue(); -} +skipTo((getCurrentValue() ~BLOCK_GROUP_INDEX_MASK) + MAX_BLOCKS_IN_GROUP); // Make sure there's no conflict with existing random block IDs -while (hasValidBlockInRange(super.getCurrentValue())) { - super.skipTo(super.getCurrentValue() + - HdfsConstants.MAX_BLOCKS_IN_GROUP); +final Block b = new Block(getCurrentValue()); +while (hasValidBlockInRange(b)) { + skipTo(getCurrentValue() + MAX_BLOCKS_IN_GROUP); + b.setBlockId(getCurrentValue()); } -if (super.getCurrentValue() = 0) { - BlockManager.LOG.warn(All negative block group IDs are used, + - growing into positive IDs, + - which might conflict with non-erasure coded blocks.); +if (b.getBlockId() = 0) { + throw new IllegalStateException(All
[46/50] hadoop git commit: HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao.
HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cfd03ef0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cfd03ef0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cfd03ef0 Branch: refs/heads/HDFS-7285 Commit: cfd03ef0b5db9b7b4cf25c189bd0de3bb73072e7 Parents: 17c5ed6 Author: Zhe Zhang z...@apache.org Authored: Fri Apr 24 09:30:38 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 27 10:42:55 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 ++ .../SequentialBlockGroupIdGenerator.java| 39 +++--- .../SequentialBlockIdGenerator.java | 2 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 57 +++- .../server/namenode/TestAddStripedBlocks.java | 21 5 files changed, 77 insertions(+), 45 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfd03ef0/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 9357e23..cf41a9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -128,3 +128,6 @@ HDFS-8223. Should calculate checksum for parity blocks in DFSStripedOutputStream. (Yi Liu via jing9) + +HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause +block id conflicts (Jing Zhao via Zhe Zhang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfd03ef0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java index e9e22ee..de8e379 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java @@ -19,9 +19,11 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.util.SequentialNumber; +import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_GROUP_INDEX_MASK; +import static org.apache.hadoop.hdfs.protocol.HdfsConstants.MAX_BLOCKS_IN_GROUP; + /** * Generate the next valid block group ID by incrementing the maximum block * group ID allocated so far, with the first 2^10 block group IDs reserved. @@ -34,6 +36,9 @@ import org.apache.hadoop.util.SequentialNumber; * bits (n+2) to (64-m) represent the ID of its block group, while the last m * bits represent its index of the group. The value m is determined by the * maximum number of blocks in a group (MAX_BLOCKS_IN_GROUP). + * + * Note that the {@link #nextValue()} methods requires external lock to + * guarantee IDs have no conflicts. */ @InterfaceAudience.Private public class SequentialBlockGroupIdGenerator extends SequentialNumber { @@ -47,32 +52,30 @@ public class SequentialBlockGroupIdGenerator extends SequentialNumber { @Override // NumberGenerator public long nextValue() { -// Skip to next legitimate block group ID based on the naming protocol -while (super.getCurrentValue() % HdfsConstants.MAX_BLOCKS_IN_GROUP 0) { - super.nextValue(); -} +skipTo((getCurrentValue() ~BLOCK_GROUP_INDEX_MASK) + MAX_BLOCKS_IN_GROUP); // Make sure there's no conflict with existing random block IDs -while (hasValidBlockInRange(super.getCurrentValue())) { - super.skipTo(super.getCurrentValue() + - HdfsConstants.MAX_BLOCKS_IN_GROUP); +final Block b = new Block(getCurrentValue()); +while (hasValidBlockInRange(b)) { + skipTo(getCurrentValue() + MAX_BLOCKS_IN_GROUP); + b.setBlockId(getCurrentValue()); } -if (super.getCurrentValue() = 0) { - BlockManager.LOG.warn(All negative block group IDs are used, + - growing into positive IDs, + - which might conflict with non-erasure coded blocks.); +if (b.getBlockId() = 0) { + throw new IllegalStateException(All
hadoop git commit: HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao.
Repository: hadoop Updated Branches: refs/heads/HDFS-7285 ebb467f33 - b2ba6836b HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2ba6836 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2ba6836 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2ba6836 Branch: refs/heads/HDFS-7285 Commit: b2ba6836bd49ed27a80ab3c7b5b21a810f9c4a30 Parents: ebb467f Author: Zhe Zhang z...@apache.org Authored: Fri Apr 24 09:30:38 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Fri Apr 24 09:31:51 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 ++ .../SequentialBlockGroupIdGenerator.java| 39 +++--- .../SequentialBlockIdGenerator.java | 2 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 57 +++- .../server/namenode/TestAddStripedBlocks.java | 21 5 files changed, 77 insertions(+), 45 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2ba6836/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 9357e23..cf41a9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -128,3 +128,6 @@ HDFS-8223. Should calculate checksum for parity blocks in DFSStripedOutputStream. (Yi Liu via jing9) + +HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause +block id conflicts (Jing Zhao via Zhe Zhang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2ba6836/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java index e9e22ee..de8e379 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java @@ -19,9 +19,11 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.util.SequentialNumber; +import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_GROUP_INDEX_MASK; +import static org.apache.hadoop.hdfs.protocol.HdfsConstants.MAX_BLOCKS_IN_GROUP; + /** * Generate the next valid block group ID by incrementing the maximum block * group ID allocated so far, with the first 2^10 block group IDs reserved. @@ -34,6 +36,9 @@ import org.apache.hadoop.util.SequentialNumber; * bits (n+2) to (64-m) represent the ID of its block group, while the last m * bits represent its index of the group. The value m is determined by the * maximum number of blocks in a group (MAX_BLOCKS_IN_GROUP). + * + * Note that the {@link #nextValue()} methods requires external lock to + * guarantee IDs have no conflicts. */ @InterfaceAudience.Private public class SequentialBlockGroupIdGenerator extends SequentialNumber { @@ -47,32 +52,30 @@ public class SequentialBlockGroupIdGenerator extends SequentialNumber { @Override // NumberGenerator public long nextValue() { -// Skip to next legitimate block group ID based on the naming protocol -while (super.getCurrentValue() % HdfsConstants.MAX_BLOCKS_IN_GROUP 0) { - super.nextValue(); -} +skipTo((getCurrentValue() ~BLOCK_GROUP_INDEX_MASK) + MAX_BLOCKS_IN_GROUP); // Make sure there's no conflict with existing random block IDs -while (hasValidBlockInRange(super.getCurrentValue())) { - super.skipTo(super.getCurrentValue() + - HdfsConstants.MAX_BLOCKS_IN_GROUP); +final Block b = new Block(getCurrentValue()); +while (hasValidBlockInRange(b)) { + skipTo(getCurrentValue() + MAX_BLOCKS_IN_GROUP); + b.setBlockId(getCurrentValue()); } -if (super.getCurrentValue() = 0) { - BlockManager.LOG.warn(All negative block group IDs are used, + - growing into positive IDs, + - which might conflict with non-erasure coded