[05/50] hadoop git commit: HDFS-8120. Erasure coding: created util class to analyze striped block groups. Contributed by Zhe Zhang and Li Bo.
HDFS-8120. Erasure coding: created util class to analyze striped block groups.
Contributed by Zhe Zhang and Li Bo.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed68791c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed68791c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed68791c
Branch: refs/heads/HDFS-7285
Commit: ed68791cba9898300b16543ac13f3efcdc45d3eb
Parents: 43df7be
Author: Jing Zhao ji...@apache.org
Authored: Wed Apr 15 12:59:27 2015 -0700
Committer: Zhe Zhang z...@apache.org
Committed: Mon May 11 11:36:14 2015 -0700
--
.../org/apache/hadoop/hdfs/DFSInputStream.java | 4 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 77 +++
.../hadoop/hdfs/DFSStripedOutputStream.java | 34 +++--
.../apache/hadoop/hdfs/StripedDataStreamer.java | 58 ++--
.../server/blockmanagement/BlockManager.java| 26 +++-
.../hadoop/hdfs/util/StripedBlockUtil.java | 138 +++
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 91 +++-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 83 +--
.../apache/hadoop/hdfs/TestReadStripedFile.java | 92 +++--
.../server/namenode/TestAddStripedBlocks.java | 107 ++
.../namenode/TestRecoverStripedBlocks.java | 3 +-
.../hadoop/hdfs/util/TestStripedBlockUtil.java | 125 +
12 files changed, 562 insertions(+), 276 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed68791c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 9104f84..16250dd 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -1148,9 +1148,9 @@ implements ByteBufferReadable, CanSetDropBehind,
CanSetReadahead,
for (int i = 0; i offsets.length; i++) {
int nread = reader.readAll(buf, offsets[i], lengths[i]);
updateReadStatistics(readStatistics, nread, reader);
- if (nread != len) {
+ if (nread != lengths[i]) {
throw new IOException(truncated return from reader.read(): +
-excpected + len + , got + nread);
+excpected + lengths[i] + , got + nread);
}
}
DFSClientFaultInjector.get().readFromDatanodeDelay();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed68791c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 8a431b1..d597407 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.apache.hadoop.net.NetUtils;
import org.apache.htrace.Span;
import org.apache.htrace.Trace;
@@ -50,7 +51,7 @@ import java.util.concurrent.Future;
*
* | - Striped Block Group - |
* blk_0 blk_1 blk_2 - A striped block group has
- *| | | {@link #groupSize} blocks
+ *| | | {@link #dataBlkNum} blocks
*v v v
* +--+ +--+ +--+
* |cell_0| |cell_1| |cell_2| - The logical read order should be
@@ -72,7 +73,7 @@ import java.util.concurrent.Future;
public class DFSStripedInputStream extends DFSInputStream {
/**
* This method plans the read portion from each block in the stripe
- * @param groupSize The size / width of the striping group
+ * @param dataBlkNum The number of data blocks in the striping group
* @param cellSize The size of each striping cell
* @param startInBlk Starting offset in the striped block
* @param len Length of the read request
@@ -81,29 +82,29 @@ public class DFSStripedInputStream extends DFSInputStream {
* for an individual
[22/50] hadoop git commit: HDFS-8120. Erasure coding: created util class to analyze striped block groups. Contributed by Zhe Zhang and Li Bo.
HDFS-8120. Erasure coding: created util class to analyze striped block groups.
Contributed by Zhe Zhang and Li Bo.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4d9bfb8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4d9bfb8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4d9bfb8
Branch: refs/heads/HDFS-7285
Commit: e4d9bfb8b4ab584f0abd051ae4b854613d26705f
Parents: a969f86
Author: Jing Zhao ji...@apache.org
Authored: Wed Apr 15 12:59:27 2015 -0700
Committer: Zhe Zhang z...@apache.org
Committed: Mon May 4 10:13:25 2015 -0700
--
.../org/apache/hadoop/hdfs/DFSInputStream.java | 4 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 77 +++
.../hadoop/hdfs/DFSStripedOutputStream.java | 34 +++--
.../apache/hadoop/hdfs/StripedDataStreamer.java | 58 ++--
.../server/blockmanagement/BlockManager.java| 26 +++-
.../hadoop/hdfs/util/StripedBlockUtil.java | 138 +++
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 91 +++-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 83 +--
.../apache/hadoop/hdfs/TestReadStripedFile.java | 92 +++--
.../server/namenode/TestAddStripedBlocks.java | 107 ++
.../namenode/TestRecoverStripedBlocks.java | 3 +-
.../hadoop/hdfs/util/TestStripedBlockUtil.java | 125 +
12 files changed, 562 insertions(+), 276 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4d9bfb8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 9104f84..16250dd 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -1148,9 +1148,9 @@ implements ByteBufferReadable, CanSetDropBehind,
CanSetReadahead,
for (int i = 0; i offsets.length; i++) {
int nread = reader.readAll(buf, offsets[i], lengths[i]);
updateReadStatistics(readStatistics, nread, reader);
- if (nread != len) {
+ if (nread != lengths[i]) {
throw new IOException(truncated return from reader.read(): +
-excpected + len + , got + nread);
+excpected + lengths[i] + , got + nread);
}
}
DFSClientFaultInjector.get().readFromDatanodeDelay();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4d9bfb8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 8a431b1..d597407 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.apache.hadoop.net.NetUtils;
import org.apache.htrace.Span;
import org.apache.htrace.Trace;
@@ -50,7 +51,7 @@ import java.util.concurrent.Future;
*
* | - Striped Block Group - |
* blk_0 blk_1 blk_2 - A striped block group has
- *| | | {@link #groupSize} blocks
+ *| | | {@link #dataBlkNum} blocks
*v v v
* +--+ +--+ +--+
* |cell_0| |cell_1| |cell_2| - The logical read order should be
@@ -72,7 +73,7 @@ import java.util.concurrent.Future;
public class DFSStripedInputStream extends DFSInputStream {
/**
* This method plans the read portion from each block in the stripe
- * @param groupSize The size / width of the striping group
+ * @param dataBlkNum The number of data blocks in the striping group
* @param cellSize The size of each striping cell
* @param startInBlk Starting offset in the striped block
* @param len Length of the read request
@@ -81,29 +82,29 @@ public class DFSStripedInputStream extends DFSInputStream {
* for an individual block
[26/50] hadoop git commit: HDFS-8120. Erasure coding: created util class to analyze striped block groups. Contributed by Zhe Zhang and Li Bo.
HDFS-8120. Erasure coding: created util class to analyze striped block groups.
Contributed by Zhe Zhang and Li Bo.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d27f5dce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d27f5dce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d27f5dce
Branch: refs/heads/HDFS-7285
Commit: d27f5dce8f7cfc7eff009817de65c1acdcf254ca
Parents: 186f139
Author: Jing Zhao ji...@apache.org
Authored: Wed Apr 15 12:59:27 2015 -0700
Committer: Jing Zhao ji...@apache.org
Committed: Wed Apr 29 11:16:53 2015 -0700
--
.../org/apache/hadoop/hdfs/DFSInputStream.java | 4 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 77 +++
.../hadoop/hdfs/DFSStripedOutputStream.java | 34 +++--
.../apache/hadoop/hdfs/StripedDataStreamer.java | 58 ++--
.../server/blockmanagement/BlockManager.java| 26 +++-
.../hadoop/hdfs/util/StripedBlockUtil.java | 138 +++
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 91 +++-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 83 +--
.../apache/hadoop/hdfs/TestReadStripedFile.java | 92 +++--
.../server/namenode/TestAddStripedBlocks.java | 107 ++
.../namenode/TestRecoverStripedBlocks.java | 3 +-
.../hadoop/hdfs/util/TestStripedBlockUtil.java | 125 +
12 files changed, 562 insertions(+), 276 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d27f5dce/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 9104f84..16250dd 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -1148,9 +1148,9 @@ implements ByteBufferReadable, CanSetDropBehind,
CanSetReadahead,
for (int i = 0; i offsets.length; i++) {
int nread = reader.readAll(buf, offsets[i], lengths[i]);
updateReadStatistics(readStatistics, nread, reader);
- if (nread != len) {
+ if (nread != lengths[i]) {
throw new IOException(truncated return from reader.read(): +
-excpected + len + , got + nread);
+excpected + lengths[i] + , got + nread);
}
}
DFSClientFaultInjector.get().readFromDatanodeDelay();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d27f5dce/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 8a431b1..d597407 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.apache.hadoop.net.NetUtils;
import org.apache.htrace.Span;
import org.apache.htrace.Trace;
@@ -50,7 +51,7 @@ import java.util.concurrent.Future;
*
* | - Striped Block Group - |
* blk_0 blk_1 blk_2 - A striped block group has
- *| | | {@link #groupSize} blocks
+ *| | | {@link #dataBlkNum} blocks
*v v v
* +--+ +--+ +--+
* |cell_0| |cell_1| |cell_2| - The logical read order should be
@@ -72,7 +73,7 @@ import java.util.concurrent.Future;
public class DFSStripedInputStream extends DFSInputStream {
/**
* This method plans the read portion from each block in the stripe
- * @param groupSize The size / width of the striping group
+ * @param dataBlkNum The number of data blocks in the striping group
* @param cellSize The size of each striping cell
* @param startInBlk Starting offset in the striped block
* @param len Length of the read request
@@ -81,29 +82,29 @@ public class DFSStripedInputStream extends DFSInputStream {
* for an individual
[30/50] hadoop git commit: HDFS-8120. Erasure coding: created util class to analyze striped block groups. Contributed by Zhe Zhang and Li Bo.
HDFS-8120. Erasure coding: created util class to analyze striped block groups.
Contributed by Zhe Zhang and Li Bo.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3fb010a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3fb010a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3fb010a
Branch: refs/heads/HDFS-7285
Commit: b3fb010a90d9e631406519d2e29c29981ee15f81
Parents: 3e97c58
Author: Jing Zhao ji...@apache.org
Authored: Wed Apr 15 12:59:27 2015 -0700
Committer: Zhe Zhang z...@apache.org
Committed: Mon Apr 27 10:42:34 2015 -0700
--
.../org/apache/hadoop/hdfs/DFSInputStream.java | 4 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 77 +++
.../hadoop/hdfs/DFSStripedOutputStream.java | 34 +++--
.../apache/hadoop/hdfs/StripedDataStreamer.java | 58 ++--
.../server/blockmanagement/BlockManager.java| 26 +++-
.../hadoop/hdfs/util/StripedBlockUtil.java | 138 +++
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 91 +++-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 83 +--
.../apache/hadoop/hdfs/TestReadStripedFile.java | 92 +++--
.../server/namenode/TestAddStripedBlocks.java | 107 ++
.../namenode/TestRecoverStripedBlocks.java | 3 +-
.../hadoop/hdfs/util/TestStripedBlockUtil.java | 125 +
12 files changed, 562 insertions(+), 276 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3fb010a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index d728fda..705e0b7 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -1169,9 +1169,9 @@ implements ByteBufferReadable, CanSetDropBehind,
CanSetReadahead,
int nread = reader.readAll(buf, offsets[i], lengths[i]);
updateReadStatistics(readStatistics, nread, reader);
- if (nread != len) {
+ if (nread != lengths[i]) {
throw new IOException(truncated return from reader.read(): +
-excpected + len + , got + nread);
+excpected + lengths[i] + , got + nread);
}
}
DFSClientFaultInjector.get().readFromDatanodeDelay();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3fb010a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 8a431b1..d597407 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.apache.hadoop.net.NetUtils;
import org.apache.htrace.Span;
import org.apache.htrace.Trace;
@@ -50,7 +51,7 @@ import java.util.concurrent.Future;
*
* | - Striped Block Group - |
* blk_0 blk_1 blk_2 - A striped block group has
- *| | | {@link #groupSize} blocks
+ *| | | {@link #dataBlkNum} blocks
*v v v
* +--+ +--+ +--+
* |cell_0| |cell_1| |cell_2| - The logical read order should be
@@ -72,7 +73,7 @@ import java.util.concurrent.Future;
public class DFSStripedInputStream extends DFSInputStream {
/**
* This method plans the read portion from each block in the stripe
- * @param groupSize The size / width of the striping group
+ * @param dataBlkNum The number of data blocks in the striping group
* @param cellSize The size of each striping cell
* @param startInBlk Starting offset in the striped block
* @param len Length of the read request
@@ -81,29 +82,29 @@ public class DFSStripedInputStream extends DFSInputStream {
* for an individual block in the group
*/
@VisibleForTesting
-
[44/50] hadoop git commit: HDFS-8120. Erasure coding: created util class to analyze striped block groups. Contributed by Zhe Zhang and Li Bo.
HDFS-8120. Erasure coding: created util class to analyze striped block groups.
Contributed by Zhe Zhang and Li Bo.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff7de640
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff7de640
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff7de640
Branch: refs/heads/HDFS-7285
Commit: ff7de64055b21d6557c1f7941e076ac9c2465171
Parents: 58cf954
Author: Jing Zhao ji...@apache.org
Authored: Wed Apr 15 12:59:27 2015 -0700
Committer: Zhe Zhang z...@apache.org
Committed: Mon Apr 20 10:28:40 2015 -0700
--
.../org/apache/hadoop/hdfs/DFSInputStream.java | 4 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 77 +++
.../hadoop/hdfs/DFSStripedOutputStream.java | 34 +++--
.../apache/hadoop/hdfs/StripedDataStreamer.java | 58 ++--
.../server/blockmanagement/BlockManager.java| 26 +++-
.../hadoop/hdfs/util/StripedBlockUtil.java | 138 +++
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 91 +++-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 83 +--
.../apache/hadoop/hdfs/TestReadStripedFile.java | 92 +++--
.../server/namenode/TestAddStripedBlocks.java | 107 ++
.../namenode/TestRecoverStripedBlocks.java | 3 +-
.../hadoop/hdfs/util/TestStripedBlockUtil.java | 125 +
12 files changed, 562 insertions(+), 276 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff7de640/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index d728fda..705e0b7 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -1169,9 +1169,9 @@ implements ByteBufferReadable, CanSetDropBehind,
CanSetReadahead,
int nread = reader.readAll(buf, offsets[i], lengths[i]);
updateReadStatistics(readStatistics, nread, reader);
- if (nread != len) {
+ if (nread != lengths[i]) {
throw new IOException(truncated return from reader.read(): +
-excpected + len + , got + nread);
+excpected + lengths[i] + , got + nread);
}
}
DFSClientFaultInjector.get().readFromDatanodeDelay();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff7de640/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 8a431b1..d597407 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.apache.hadoop.net.NetUtils;
import org.apache.htrace.Span;
import org.apache.htrace.Trace;
@@ -50,7 +51,7 @@ import java.util.concurrent.Future;
*
* | - Striped Block Group - |
* blk_0 blk_1 blk_2 - A striped block group has
- *| | | {@link #groupSize} blocks
+ *| | | {@link #dataBlkNum} blocks
*v v v
* +--+ +--+ +--+
* |cell_0| |cell_1| |cell_2| - The logical read order should be
@@ -72,7 +73,7 @@ import java.util.concurrent.Future;
public class DFSStripedInputStream extends DFSInputStream {
/**
* This method plans the read portion from each block in the stripe
- * @param groupSize The size / width of the striping group
+ * @param dataBlkNum The number of data blocks in the striping group
* @param cellSize The size of each striping cell
* @param startInBlk Starting offset in the striped block
* @param len Length of the read request
@@ -81,29 +82,29 @@ public class DFSStripedInputStream extends DFSInputStream {
* for an individual block in the group
*/
@VisibleForTesting
-
hadoop git commit: HDFS-8120. Erasure coding: created util class to analyze striped block groups. Contributed by Zhe Zhang and Li Bo.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 295ccbca7 - 903bf3700
HDFS-8120. Erasure coding: created util class to analyze striped block groups.
Contributed by Zhe Zhang and Li Bo.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/903bf370
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/903bf370
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/903bf370
Branch: refs/heads/HDFS-7285
Commit: 903bf37006630185581c4b4e3d11283c36a61aab
Parents: 295ccbc
Author: Jing Zhao ji...@apache.org
Authored: Wed Apr 15 12:59:27 2015 -0700
Committer: Jing Zhao ji...@apache.org
Committed: Wed Apr 15 12:59:27 2015 -0700
--
.../org/apache/hadoop/hdfs/DFSInputStream.java | 4 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 77 +++
.../hadoop/hdfs/DFSStripedOutputStream.java | 34 +++--
.../apache/hadoop/hdfs/StripedDataStreamer.java | 58 ++--
.../server/blockmanagement/BlockManager.java| 26 +++-
.../hadoop/hdfs/util/StripedBlockUtil.java | 138 +++
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 91 +++-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 83 +--
.../apache/hadoop/hdfs/TestReadStripedFile.java | 92 +++--
.../server/namenode/TestAddStripedBlocks.java | 107 ++
.../namenode/TestRecoverStripedBlocks.java | 3 +-
.../hadoop/hdfs/util/TestStripedBlockUtil.java | 125 +
12 files changed, 562 insertions(+), 276 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/903bf370/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 6006693..9ef 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -1169,9 +1169,9 @@ implements ByteBufferReadable, CanSetDropBehind,
CanSetReadahead,
int nread = reader.readAll(buf, offsets[i], lengths[i]);
updateReadStatistics(readStatistics, nread, reader);
- if (nread != len) {
+ if (nread != lengths[i]) {
throw new IOException(truncated return from reader.read(): +
-excpected + len + , got + nread);
+excpected + lengths[i] + , got + nread);
}
}
DFSClientFaultInjector.get().readFromDatanodeDelay();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/903bf370/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 8a431b1..d597407 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.apache.hadoop.net.NetUtils;
import org.apache.htrace.Span;
import org.apache.htrace.Trace;
@@ -50,7 +51,7 @@ import java.util.concurrent.Future;
*
* | - Striped Block Group - |
* blk_0 blk_1 blk_2 - A striped block group has
- *| | | {@link #groupSize} blocks
+ *| | | {@link #dataBlkNum} blocks
*v v v
* +--+ +--+ +--+
* |cell_0| |cell_1| |cell_2| - The logical read order should be
@@ -72,7 +73,7 @@ import java.util.concurrent.Future;
public class DFSStripedInputStream extends DFSInputStream {
/**
* This method plans the read portion from each block in the stripe
- * @param groupSize The size / width of the striping group
+ * @param dataBlkNum The number of data blocks in the striping group
* @param cellSize The size of each striping cell
* @param startInBlk Starting offset in the striped block
* @param len Length of the read request
@@ -81,29 +82,29 @@ public class DFSStripedInputStream extends DFSInputStream {
