[34/50] hadoop git commit: HDFS-8190. StripedBlockUtil.getInternalBlockLength may have overflow error.
HDFS-8190. StripedBlockUtil.getInternalBlockLength may have overflow error.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e875f1dc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e875f1dc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e875f1dc
Branch: refs/heads/HDFS-7285
Commit: e875f1dcbc64a297dad2c654cec7470ff3b5084d
Parents: 08b56e4
Author: Tsz-Wo Nicholas Sze szets...@hortonworks.com
Authored: Mon Apr 20 17:42:02 2015 -0700
Committer: Zhe Zhang z...@apache.org
Committed: Mon May 4 10:13:28 2015 -0700
--
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 +
.../hadoop/hdfs/util/StripedBlockUtil.java | 61 ---
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 178 +++
3 files changed, 100 insertions(+), 142 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e875f1dc/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index c8dbf08..8f28285 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -104,3 +104,6 @@
HDFS-8181. createErasureCodingZone sets retryCache state as false always
(Uma Maheswara Rao G via vinayakumarb)
+
+HDFS-8190. StripedBlockUtil.getInternalBlockLength may have overflow error.
+(szetszwo)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e875f1dc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
index 2368021..d622d4d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
@@ -25,6 +25,8 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import com.google.common.base.Preconditions;
+
/**
* Utility class for analyzing striped block groups
*/
@@ -81,46 +83,43 @@ public class StripedBlockUtil {
/**
* Get the size of an internal block at the given index of a block group
*
- * @param numBytesInGroup Size of the block group only counting data blocks
+ * @param dataSize Size of the block group only counting data blocks
* @param cellSize The size of a striping cell
- * @param dataBlkNum The number of data blocks
- * @param idxInGroup The logical index in the striped block group
+ * @param numDataBlocks The number of data blocks
+ * @param i The logical index in the striped block group
* @return The size of the internal block at the specified index
*/
- public static long getInternalBlockLength(long numBytesInGroup,
- int cellSize, int dataBlkNum, int idxInGroup) {
+ public static long getInternalBlockLength(long dataSize,
+ int cellSize, int numDataBlocks, int i) {
+Preconditions.checkArgument(dataSize = 0);
+Preconditions.checkArgument(cellSize 0);
+Preconditions.checkArgument(numDataBlocks 0);
+Preconditions.checkArgument(i = 0);
// Size of each stripe (only counting data blocks)
-final long numBytesPerStripe = cellSize * dataBlkNum;
-assert numBytesPerStripe 0:
-getInternalBlockLength should only be called on valid striped blocks;
+final int stripeSize = cellSize * numDataBlocks;
// If block group ends at stripe boundary, each internal block has an equal
// share of the group
-if (numBytesInGroup % numBytesPerStripe == 0) {
- return numBytesInGroup / dataBlkNum;
+final int lastStripeDataLen = (int)(dataSize % stripeSize);
+if (lastStripeDataLen == 0) {
+ return dataSize / numDataBlocks;
}
-int numStripes = (int) ((numBytesInGroup - 1) / numBytesPerStripe + 1);
-assert numStripes = 1 : There should be at least 1 stripe;
-
-// All stripes but the last one are full stripes. The block should at least
-// contain (numStripes - 1) full cells.
-long blkSize = (numStripes - 1) * cellSize;
-
-long lastStripeLen = numBytesInGroup % numBytesPerStripe;
-// Size of parity cells should equal the size of the first cell, if it
-// is not full.
-long lastParityCellLen = Math.min(cellSize, lastStripeLen);
-
-if (idxInGroup = dataBlkNum) {
- // for
[34/50] hadoop git commit: HDFS-8190. StripedBlockUtil.getInternalBlockLength may have overflow error.
HDFS-8190. StripedBlockUtil.getInternalBlockLength may have overflow error.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b0ad404
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b0ad404
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b0ad404
Branch: refs/heads/HDFS-7285
Commit: 4b0ad40441f6ea56f63c57b371d1837547b2b2b1
Parents: df77601
Author: Tsz-Wo Nicholas Sze szets...@hortonworks.com
Authored: Mon Apr 20 17:42:02 2015 -0700
Committer: Jing Zhao ji...@apache.org
Committed: Wed Apr 29 11:16:55 2015 -0700
--
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 +
.../hadoop/hdfs/util/StripedBlockUtil.java | 61 ---
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 178 +++
3 files changed, 100 insertions(+), 142 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b0ad404/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index c8dbf08..8f28285 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -104,3 +104,6 @@
HDFS-8181. createErasureCodingZone sets retryCache state as false always
(Uma Maheswara Rao G via vinayakumarb)
+
+HDFS-8190. StripedBlockUtil.getInternalBlockLength may have overflow error.
+(szetszwo)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b0ad404/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
index 2368021..d622d4d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
@@ -25,6 +25,8 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import com.google.common.base.Preconditions;
+
/**
* Utility class for analyzing striped block groups
*/
@@ -81,46 +83,43 @@ public class StripedBlockUtil {
/**
* Get the size of an internal block at the given index of a block group
*
- * @param numBytesInGroup Size of the block group only counting data blocks
+ * @param dataSize Size of the block group only counting data blocks
* @param cellSize The size of a striping cell
- * @param dataBlkNum The number of data blocks
- * @param idxInGroup The logical index in the striped block group
+ * @param numDataBlocks The number of data blocks
+ * @param i The logical index in the striped block group
* @return The size of the internal block at the specified index
*/
- public static long getInternalBlockLength(long numBytesInGroup,
- int cellSize, int dataBlkNum, int idxInGroup) {
+ public static long getInternalBlockLength(long dataSize,
+ int cellSize, int numDataBlocks, int i) {
+Preconditions.checkArgument(dataSize = 0);
+Preconditions.checkArgument(cellSize 0);
+Preconditions.checkArgument(numDataBlocks 0);
+Preconditions.checkArgument(i = 0);
// Size of each stripe (only counting data blocks)
-final long numBytesPerStripe = cellSize * dataBlkNum;
-assert numBytesPerStripe 0:
-getInternalBlockLength should only be called on valid striped blocks;
+final int stripeSize = cellSize * numDataBlocks;
// If block group ends at stripe boundary, each internal block has an equal
// share of the group
-if (numBytesInGroup % numBytesPerStripe == 0) {
- return numBytesInGroup / dataBlkNum;
+final int lastStripeDataLen = (int)(dataSize % stripeSize);
+if (lastStripeDataLen == 0) {
+ return dataSize / numDataBlocks;
}
-int numStripes = (int) ((numBytesInGroup - 1) / numBytesPerStripe + 1);
-assert numStripes = 1 : There should be at least 1 stripe;
-
-// All stripes but the last one are full stripes. The block should at least
-// contain (numStripes - 1) full cells.
-long blkSize = (numStripes - 1) * cellSize;
-
-long lastStripeLen = numBytesInGroup % numBytesPerStripe;
-// Size of parity cells should equal the size of the first cell, if it
-// is not full.
-long lastParityCellLen = Math.min(cellSize, lastStripeLen);
-
-if (idxInGroup = dataBlkNum) {
- // for
