subject:"\[41\/50\] hadoop git commit\: HDFS\-8228. Erasure Coding\: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao."

[41/50] hadoop git commit: HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao.

2015-05-04 Thread zhz

HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause 
block id conflicts. Contributed by Jing Zhao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/395e29bb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/395e29bb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/395e29bb

Branch: refs/heads/HDFS-7285
Commit: 395e29bb0bf4f46722a0e5cea0bffba64052edde
Parents: 254759d
Author: Zhe Zhang z...@apache.org
Authored: Fri Apr 24 09:30:38 2015 -0700
Committer: Zhe Zhang z...@apache.org
Committed: Mon May 4 10:13:30 2015 -0700

--
 .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt|  3 ++
 .../SequentialBlockGroupIdGenerator.java| 39 +++---
 .../SequentialBlockIdGenerator.java |  2 +-
 .../hadoop/hdfs/TestDFSStripedInputStream.java  | 57 +++-
 .../server/namenode/TestAddStripedBlocks.java   | 21 
 5 files changed, 77 insertions(+), 45 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/395e29bb/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 9357e23..cf41a9b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -128,3 +128,6 @@
 
 HDFS-8223. Should calculate checksum for parity blocks in 
DFSStripedOutputStream.
 (Yi Liu via jing9)
+
+HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may 
cause 
+block id conflicts (Jing Zhao via Zhe Zhang)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/395e29bb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
index e9e22ee..de8e379 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
@@ -19,9 +19,11 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.util.SequentialNumber;
 
+import static 
org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_GROUP_INDEX_MASK;
+import static 
org.apache.hadoop.hdfs.protocol.HdfsConstants.MAX_BLOCKS_IN_GROUP;
+
 /**
  * Generate the next valid block group ID by incrementing the maximum block
  * group ID allocated so far, with the first 2^10 block group IDs reserved.
@@ -34,6 +36,9 @@ import org.apache.hadoop.util.SequentialNumber;
  * bits (n+2) to (64-m) represent the ID of its block group, while the last m
  * bits represent its index of the group. The value m is determined by the
  * maximum number of blocks in a group (MAX_BLOCKS_IN_GROUP).
+ *
+ * Note that the {@link #nextValue()} methods requires external lock to
+ * guarantee IDs have no conflicts.
  */
 @InterfaceAudience.Private
 public class SequentialBlockGroupIdGenerator extends SequentialNumber {
@@ -47,32 +52,30 @@ public class SequentialBlockGroupIdGenerator extends 
SequentialNumber {
 
   @Override // NumberGenerator
   public long nextValue() {
-// Skip to next legitimate block group ID based on the naming protocol
-while (super.getCurrentValue() % HdfsConstants.MAX_BLOCKS_IN_GROUP  0) {
-  super.nextValue();
-}
+skipTo((getCurrentValue()  ~BLOCK_GROUP_INDEX_MASK) + 
MAX_BLOCKS_IN_GROUP);
 // Make sure there's no conflict with existing random block IDs
-while (hasValidBlockInRange(super.getCurrentValue())) {
-  super.skipTo(super.getCurrentValue() +
-  HdfsConstants.MAX_BLOCKS_IN_GROUP);
+final Block b = new Block(getCurrentValue());
+while (hasValidBlockInRange(b)) {
+  skipTo(getCurrentValue() + MAX_BLOCKS_IN_GROUP);
+  b.setBlockId(getCurrentValue());
 }
-if (super.getCurrentValue() = 0) {
-  BlockManager.LOG.warn(All negative block group IDs are used,  +
-  growing into positive IDs,  +
-  which might conflict with non-erasure coded blocks.);
+if (b.getBlockId() = 0) {
+  throw new IllegalStateException(All

[41/50] hadoop git commit: HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao.

2015-04-29 Thread jing9

HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause 
block id conflicts. Contributed by Jing Zhao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1e7dfa0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1e7dfa0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1e7dfa0

Branch: refs/heads/HDFS-7285
Commit: d1e7dfa0e0e9bf76f4925936f27b1768339cb5dd
Parents: f24c2c0
Author: Zhe Zhang z...@apache.org
Authored: Fri Apr 24 09:30:38 2015 -0700
Committer: Jing Zhao ji...@apache.org
Committed: Wed Apr 29 11:16:56 2015 -0700

--
 .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt|  3 ++
 .../SequentialBlockGroupIdGenerator.java| 39 +++---
 .../SequentialBlockIdGenerator.java |  2 +-
 .../hadoop/hdfs/TestDFSStripedInputStream.java  | 57 +++-
 .../server/namenode/TestAddStripedBlocks.java   | 21 
 5 files changed, 77 insertions(+), 45 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1e7dfa0/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 9357e23..cf41a9b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -128,3 +128,6 @@
 
 HDFS-8223. Should calculate checksum for parity blocks in 
DFSStripedOutputStream.
 (Yi Liu via jing9)
+
+HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may 
cause 
+block id conflicts (Jing Zhao via Zhe Zhang)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1e7dfa0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
index e9e22ee..de8e379 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
@@ -19,9 +19,11 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.util.SequentialNumber;
 
+import static 
org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_GROUP_INDEX_MASK;
+import static 
org.apache.hadoop.hdfs.protocol.HdfsConstants.MAX_BLOCKS_IN_GROUP;
+
 /**
  * Generate the next valid block group ID by incrementing the maximum block
  * group ID allocated so far, with the first 2^10 block group IDs reserved.
@@ -34,6 +36,9 @@ import org.apache.hadoop.util.SequentialNumber;
  * bits (n+2) to (64-m) represent the ID of its block group, while the last m
  * bits represent its index of the group. The value m is determined by the
  * maximum number of blocks in a group (MAX_BLOCKS_IN_GROUP).
+ *
+ * Note that the {@link #nextValue()} methods requires external lock to
+ * guarantee IDs have no conflicts.
  */
 @InterfaceAudience.Private
 public class SequentialBlockGroupIdGenerator extends SequentialNumber {
@@ -47,32 +52,30 @@ public class SequentialBlockGroupIdGenerator extends 
SequentialNumber {
 
   @Override // NumberGenerator
   public long nextValue() {
-// Skip to next legitimate block group ID based on the naming protocol
-while (super.getCurrentValue() % HdfsConstants.MAX_BLOCKS_IN_GROUP  0) {
-  super.nextValue();
-}
+skipTo((getCurrentValue()  ~BLOCK_GROUP_INDEX_MASK) + 
MAX_BLOCKS_IN_GROUP);
 // Make sure there's no conflict with existing random block IDs
-while (hasValidBlockInRange(super.getCurrentValue())) {
-  super.skipTo(super.getCurrentValue() +
-  HdfsConstants.MAX_BLOCKS_IN_GROUP);
+final Block b = new Block(getCurrentValue());
+while (hasValidBlockInRange(b)) {
+  skipTo(getCurrentValue() + MAX_BLOCKS_IN_GROUP);
+  b.setBlockId(getCurrentValue());
 }
-if (super.getCurrentValue() = 0) {
-  BlockManager.LOG.warn(All negative block group IDs are used,  +
-  growing into positive IDs,  +
-  which might conflict with non-erasure coded blocks.);
+if (b.getBlockId() = 0) {
+  throw new IllegalStateException(All

[41/50] hadoop git commit: HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao.

[41/50] hadoop git commit: HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts. Contributed by Jing Zhao.

2 matches

Site Navigation

Mail list logo

Footer information