[13/50] hadoop git commit: HDFS-8033. Erasure coding: stateful (non-positional) read from files in striped layout. Contributed by Zhe Zhang.
HDFS-8033. Erasure coding: stateful (non-positional) read from files in striped layout. Contributed by Zhe Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c79ec3c4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c79ec3c4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c79ec3c4 Branch: refs/heads/HDFS-7285 Commit: c79ec3c467a5e275a6a4e0635560df946d3055a6 Parents: 98ebb4e Author: Zhe Zhang z...@apache.org Authored: Fri Apr 24 22:36:15 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon May 18 10:01:48 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 + .../org/apache/hadoop/hdfs/DFSInputStream.java | 55 ++-- .../hadoop/hdfs/DFSStripedInputStream.java | 311 ++- .../hadoop/hdfs/TestDFSStripedInputStream.java | 43 +++ .../apache/hadoop/hdfs/TestReadStripedFile.java | 110 ++- 5 files changed, 465 insertions(+), 57 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c79ec3c4/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index cf41a9b..e8db485 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -131,3 +131,6 @@ HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts (Jing Zhao via Zhe Zhang) + +HDFS-8033. Erasure coding: stateful (non-positional) read from files in +striped layout (Zhe Zhang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/c79ec3c4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 16250dd..6eb25d0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -95,34 +95,34 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, public static boolean tcpReadsDisabledForTesting = false; private long hedgedReadOpsLoopNumForTesting = 0; protected final DFSClient dfsClient; - private AtomicBoolean closed = new AtomicBoolean(false); - private final String src; - private final boolean verifyChecksum; + protected AtomicBoolean closed = new AtomicBoolean(false); + protected final String src; + protected final boolean verifyChecksum; // state by stateful read only: // (protected by lock on this) / private DatanodeInfo currentNode = null; - private LocatedBlock currentLocatedBlock = null; - private long pos = 0; - private long blockEnd = -1; + protected LocatedBlock currentLocatedBlock = null; + protected long pos = 0; + protected long blockEnd = -1; private BlockReader blockReader = null; // state shared by stateful and positional read: // (protected by lock on infoLock) - private LocatedBlocks locatedBlocks = null; + protected LocatedBlocks locatedBlocks = null; private long lastBlockBeingWrittenLength = 0; private FileEncryptionInfo fileEncryptionInfo = null; - private CachingStrategy cachingStrategy; + protected CachingStrategy cachingStrategy; - private final ReadStatistics readStatistics = new ReadStatistics(); + protected final ReadStatistics readStatistics = new ReadStatistics(); // lock for state shared between read and pread // Note: Never acquire a lock on this with this lock held to avoid deadlocks // (it's OK to acquire this lock when the lock on this is held) - private final Object infoLock = new Object(); + protected final Object infoLock = new Object(); /** * Track the ByteBuffers that we have handed out to readers. @@ -239,7 +239,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, * back to the namenode to get a new list of block locations, and is * capped at maxBlockAcquireFailures */ - private int failures = 0; + protected int failures = 0; /* XXX Use of CocurrentHashMap is temp fix. Need to fix * parallel accesses to DFSInputStream (through ptreads) properly */ @@ -476,7 +476,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, } /** Fetch a block from namenode and cache it */ - private void fetchBlockAt(long offset) throws IOException { + protected void fetchBlockAt(long
[15/50] hadoop git commit: HDFS-8033. Erasure coding: stateful (non-positional) read from files in striped layout. Contributed by Zhe Zhang.
HDFS-8033. Erasure coding: stateful (non-positional) read from files in striped layout. Contributed by Zhe Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f37fd1f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f37fd1f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f37fd1f Branch: refs/heads/HDFS-7285 Commit: 3f37fd1fde12396269fc8510e1d8171c1c7f9211 Parents: 851b114 Author: Zhe Zhang z...@apache.org Authored: Fri Apr 24 22:36:15 2015 -0700 Committer: Jing Zhao ji...@apache.org Committed: Mon May 18 22:11:06 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 + .../org/apache/hadoop/hdfs/DFSInputStream.java | 55 ++-- .../hadoop/hdfs/DFSStripedInputStream.java | 311 ++- .../hadoop/hdfs/TestDFSStripedInputStream.java | 43 +++ .../apache/hadoop/hdfs/TestReadStripedFile.java | 110 ++- 5 files changed, 465 insertions(+), 57 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f37fd1f/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index cf41a9b..e8db485 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -131,3 +131,6 @@ HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts (Jing Zhao via Zhe Zhang) + +HDFS-8033. Erasure coding: stateful (non-positional) read from files in +striped layout (Zhe Zhang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f37fd1f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 16250dd..6eb25d0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -95,34 +95,34 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, public static boolean tcpReadsDisabledForTesting = false; private long hedgedReadOpsLoopNumForTesting = 0; protected final DFSClient dfsClient; - private AtomicBoolean closed = new AtomicBoolean(false); - private final String src; - private final boolean verifyChecksum; + protected AtomicBoolean closed = new AtomicBoolean(false); + protected final String src; + protected final boolean verifyChecksum; // state by stateful read only: // (protected by lock on this) / private DatanodeInfo currentNode = null; - private LocatedBlock currentLocatedBlock = null; - private long pos = 0; - private long blockEnd = -1; + protected LocatedBlock currentLocatedBlock = null; + protected long pos = 0; + protected long blockEnd = -1; private BlockReader blockReader = null; // state shared by stateful and positional read: // (protected by lock on infoLock) - private LocatedBlocks locatedBlocks = null; + protected LocatedBlocks locatedBlocks = null; private long lastBlockBeingWrittenLength = 0; private FileEncryptionInfo fileEncryptionInfo = null; - private CachingStrategy cachingStrategy; + protected CachingStrategy cachingStrategy; - private final ReadStatistics readStatistics = new ReadStatistics(); + protected final ReadStatistics readStatistics = new ReadStatistics(); // lock for state shared between read and pread // Note: Never acquire a lock on this with this lock held to avoid deadlocks // (it's OK to acquire this lock when the lock on this is held) - private final Object infoLock = new Object(); + protected final Object infoLock = new Object(); /** * Track the ByteBuffers that we have handed out to readers. @@ -239,7 +239,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, * back to the namenode to get a new list of block locations, and is * capped at maxBlockAcquireFailures */ - private int failures = 0; + protected int failures = 0; /* XXX Use of CocurrentHashMap is temp fix. Need to fix * parallel accesses to DFSInputStream (through ptreads) properly */ @@ -476,7 +476,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, } /** Fetch a block from namenode and cache it */ - private void fetchBlockAt(long offset) throws IOException { + protected void fetchBlockAt(long
[15/50] hadoop git commit: HDFS-8033. Erasure coding: stateful (non-positional) read from files in striped layout. Contributed by Zhe Zhang.
HDFS-8033. Erasure coding: stateful (non-positional) read from files in striped layout. Contributed by Zhe Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d8417a50 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d8417a50 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d8417a50 Branch: refs/heads/HDFS-7285 Commit: d8417a50bd0418b16b489065670cbdf4cab1cfd3 Parents: f5c6ec6 Author: Zhe Zhang z...@apache.org Authored: Fri Apr 24 22:36:15 2015 -0700 Committer: Jing Zhao ji...@apache.org Committed: Sat May 16 15:15:23 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 + .../org/apache/hadoop/hdfs/DFSInputStream.java | 55 ++-- .../hadoop/hdfs/DFSStripedInputStream.java | 311 ++- .../hadoop/hdfs/TestDFSStripedInputStream.java | 43 +++ .../apache/hadoop/hdfs/TestReadStripedFile.java | 110 ++- 5 files changed, 465 insertions(+), 57 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8417a50/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index cf41a9b..e8db485 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -131,3 +131,6 @@ HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts (Jing Zhao via Zhe Zhang) + +HDFS-8033. Erasure coding: stateful (non-positional) read from files in +striped layout (Zhe Zhang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8417a50/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 16250dd..6eb25d0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -95,34 +95,34 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, public static boolean tcpReadsDisabledForTesting = false; private long hedgedReadOpsLoopNumForTesting = 0; protected final DFSClient dfsClient; - private AtomicBoolean closed = new AtomicBoolean(false); - private final String src; - private final boolean verifyChecksum; + protected AtomicBoolean closed = new AtomicBoolean(false); + protected final String src; + protected final boolean verifyChecksum; // state by stateful read only: // (protected by lock on this) / private DatanodeInfo currentNode = null; - private LocatedBlock currentLocatedBlock = null; - private long pos = 0; - private long blockEnd = -1; + protected LocatedBlock currentLocatedBlock = null; + protected long pos = 0; + protected long blockEnd = -1; private BlockReader blockReader = null; // state shared by stateful and positional read: // (protected by lock on infoLock) - private LocatedBlocks locatedBlocks = null; + protected LocatedBlocks locatedBlocks = null; private long lastBlockBeingWrittenLength = 0; private FileEncryptionInfo fileEncryptionInfo = null; - private CachingStrategy cachingStrategy; + protected CachingStrategy cachingStrategy; - private final ReadStatistics readStatistics = new ReadStatistics(); + protected final ReadStatistics readStatistics = new ReadStatistics(); // lock for state shared between read and pread // Note: Never acquire a lock on this with this lock held to avoid deadlocks // (it's OK to acquire this lock when the lock on this is held) - private final Object infoLock = new Object(); + protected final Object infoLock = new Object(); /** * Track the ByteBuffers that we have handed out to readers. @@ -239,7 +239,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, * back to the namenode to get a new list of block locations, and is * capped at maxBlockAcquireFailures */ - private int failures = 0; + protected int failures = 0; /* XXX Use of CocurrentHashMap is temp fix. Need to fix * parallel accesses to DFSInputStream (through ptreads) properly */ @@ -476,7 +476,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, } /** Fetch a block from namenode and cache it */ - private void fetchBlockAt(long offset) throws IOException { + protected void fetchBlockAt(long
[26/50] hadoop git commit: HDFS-8033. Erasure coding: stateful (non-positional) read from files in striped layout. Contributed by Zhe Zhang.
HDFS-8033. Erasure coding: stateful (non-positional) read from files in striped layout. Contributed by Zhe Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fc1609de Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fc1609de Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fc1609de Branch: refs/heads/HDFS-7285 Commit: fc1609de4b239d664c99ea718aa48b0804266164 Parents: f2baca7 Author: Zhe Zhang z...@apache.org Authored: Fri Apr 24 22:36:15 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon May 11 11:36:20 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 + .../org/apache/hadoop/hdfs/DFSInputStream.java | 55 ++-- .../hadoop/hdfs/DFSStripedInputStream.java | 311 ++- .../hadoop/hdfs/TestDFSStripedInputStream.java | 43 +++ .../apache/hadoop/hdfs/TestReadStripedFile.java | 110 ++- 5 files changed, 465 insertions(+), 57 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc1609de/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index cf41a9b..e8db485 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -131,3 +131,6 @@ HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts (Jing Zhao via Zhe Zhang) + +HDFS-8033. Erasure coding: stateful (non-positional) read from files in +striped layout (Zhe Zhang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc1609de/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 16250dd..6eb25d0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -95,34 +95,34 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, public static boolean tcpReadsDisabledForTesting = false; private long hedgedReadOpsLoopNumForTesting = 0; protected final DFSClient dfsClient; - private AtomicBoolean closed = new AtomicBoolean(false); - private final String src; - private final boolean verifyChecksum; + protected AtomicBoolean closed = new AtomicBoolean(false); + protected final String src; + protected final boolean verifyChecksum; // state by stateful read only: // (protected by lock on this) / private DatanodeInfo currentNode = null; - private LocatedBlock currentLocatedBlock = null; - private long pos = 0; - private long blockEnd = -1; + protected LocatedBlock currentLocatedBlock = null; + protected long pos = 0; + protected long blockEnd = -1; private BlockReader blockReader = null; // state shared by stateful and positional read: // (protected by lock on infoLock) - private LocatedBlocks locatedBlocks = null; + protected LocatedBlocks locatedBlocks = null; private long lastBlockBeingWrittenLength = 0; private FileEncryptionInfo fileEncryptionInfo = null; - private CachingStrategy cachingStrategy; + protected CachingStrategy cachingStrategy; - private final ReadStatistics readStatistics = new ReadStatistics(); + protected final ReadStatistics readStatistics = new ReadStatistics(); // lock for state shared between read and pread // Note: Never acquire a lock on this with this lock held to avoid deadlocks // (it's OK to acquire this lock when the lock on this is held) - private final Object infoLock = new Object(); + protected final Object infoLock = new Object(); /** * Track the ByteBuffers that we have handed out to readers. @@ -239,7 +239,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, * back to the namenode to get a new list of block locations, and is * capped at maxBlockAcquireFailures */ - private int failures = 0; + protected int failures = 0; /* XXX Use of CocurrentHashMap is temp fix. Need to fix * parallel accesses to DFSInputStream (through ptreads) properly */ @@ -476,7 +476,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, } /** Fetch a block from namenode and cache it */ - private void fetchBlockAt(long offset) throws IOException { + protected void fetchBlockAt(long
[42/50] hadoop git commit: HDFS-8033. Erasure coding: stateful (non-positional) read from files in striped layout. Contributed by Zhe Zhang.
HDFS-8033. Erasure coding: stateful (non-positional) read from files in striped layout. Contributed by Zhe Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e51018a1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e51018a1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e51018a1 Branch: refs/heads/HDFS-7285 Commit: e51018a1b0c645071dbf1eb6ba0354b5593e8290 Parents: 395e29b Author: Zhe Zhang z...@apache.org Authored: Fri Apr 24 22:36:15 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon May 4 10:13:30 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 + .../org/apache/hadoop/hdfs/DFSInputStream.java | 55 ++-- .../hadoop/hdfs/DFSStripedInputStream.java | 311 ++- .../hadoop/hdfs/TestDFSStripedInputStream.java | 43 +++ .../apache/hadoop/hdfs/TestReadStripedFile.java | 110 ++- 5 files changed, 465 insertions(+), 57 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e51018a1/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index cf41a9b..e8db485 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -131,3 +131,6 @@ HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts (Jing Zhao via Zhe Zhang) + +HDFS-8033. Erasure coding: stateful (non-positional) read from files in +striped layout (Zhe Zhang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/e51018a1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 16250dd..6eb25d0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -95,34 +95,34 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, public static boolean tcpReadsDisabledForTesting = false; private long hedgedReadOpsLoopNumForTesting = 0; protected final DFSClient dfsClient; - private AtomicBoolean closed = new AtomicBoolean(false); - private final String src; - private final boolean verifyChecksum; + protected AtomicBoolean closed = new AtomicBoolean(false); + protected final String src; + protected final boolean verifyChecksum; // state by stateful read only: // (protected by lock on this) / private DatanodeInfo currentNode = null; - private LocatedBlock currentLocatedBlock = null; - private long pos = 0; - private long blockEnd = -1; + protected LocatedBlock currentLocatedBlock = null; + protected long pos = 0; + protected long blockEnd = -1; private BlockReader blockReader = null; // state shared by stateful and positional read: // (protected by lock on infoLock) - private LocatedBlocks locatedBlocks = null; + protected LocatedBlocks locatedBlocks = null; private long lastBlockBeingWrittenLength = 0; private FileEncryptionInfo fileEncryptionInfo = null; - private CachingStrategy cachingStrategy; + protected CachingStrategy cachingStrategy; - private final ReadStatistics readStatistics = new ReadStatistics(); + protected final ReadStatistics readStatistics = new ReadStatistics(); // lock for state shared between read and pread // Note: Never acquire a lock on this with this lock held to avoid deadlocks // (it's OK to acquire this lock when the lock on this is held) - private final Object infoLock = new Object(); + protected final Object infoLock = new Object(); /** * Track the ByteBuffers that we have handed out to readers. @@ -239,7 +239,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, * back to the namenode to get a new list of block locations, and is * capped at maxBlockAcquireFailures */ - private int failures = 0; + protected int failures = 0; /* XXX Use of CocurrentHashMap is temp fix. Need to fix * parallel accesses to DFSInputStream (through ptreads) properly */ @@ -476,7 +476,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, } /** Fetch a block from namenode and cache it */ - private void fetchBlockAt(long offset) throws IOException { + protected void fetchBlockAt(long
[45/50] hadoop git commit: HDFS-8033. Erasure coding: stateful (non-positional) read from files in striped layout. Contributed by Zhe Zhang.
HDFS-8033. Erasure coding: stateful (non-positional) read from files in striped layout. Contributed by Zhe Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f8276301 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f8276301 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f8276301 Branch: refs/heads/HDFS-7285 Commit: f8276301453cf10c6fe0e9cc01a23a61a8ab03ac Parents: d1e7dfa Author: Zhe Zhang z...@apache.org Authored: Fri Apr 24 22:36:15 2015 -0700 Committer: Jing Zhao ji...@apache.org Committed: Wed Apr 29 11:17:52 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 + .../org/apache/hadoop/hdfs/DFSInputStream.java | 55 ++-- .../hadoop/hdfs/DFSStripedInputStream.java | 311 ++- .../hadoop/hdfs/TestDFSStripedInputStream.java | 43 +++ .../apache/hadoop/hdfs/TestReadStripedFile.java | 110 ++- 5 files changed, 465 insertions(+), 57 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8276301/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index cf41a9b..e8db485 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -131,3 +131,6 @@ HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts (Jing Zhao via Zhe Zhang) + +HDFS-8033. Erasure coding: stateful (non-positional) read from files in +striped layout (Zhe Zhang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8276301/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 16250dd..6eb25d0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -95,34 +95,34 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, public static boolean tcpReadsDisabledForTesting = false; private long hedgedReadOpsLoopNumForTesting = 0; protected final DFSClient dfsClient; - private AtomicBoolean closed = new AtomicBoolean(false); - private final String src; - private final boolean verifyChecksum; + protected AtomicBoolean closed = new AtomicBoolean(false); + protected final String src; + protected final boolean verifyChecksum; // state by stateful read only: // (protected by lock on this) / private DatanodeInfo currentNode = null; - private LocatedBlock currentLocatedBlock = null; - private long pos = 0; - private long blockEnd = -1; + protected LocatedBlock currentLocatedBlock = null; + protected long pos = 0; + protected long blockEnd = -1; private BlockReader blockReader = null; // state shared by stateful and positional read: // (protected by lock on infoLock) - private LocatedBlocks locatedBlocks = null; + protected LocatedBlocks locatedBlocks = null; private long lastBlockBeingWrittenLength = 0; private FileEncryptionInfo fileEncryptionInfo = null; - private CachingStrategy cachingStrategy; + protected CachingStrategy cachingStrategy; - private final ReadStatistics readStatistics = new ReadStatistics(); + protected final ReadStatistics readStatistics = new ReadStatistics(); // lock for state shared between read and pread // Note: Never acquire a lock on this with this lock held to avoid deadlocks // (it's OK to acquire this lock when the lock on this is held) - private final Object infoLock = new Object(); + protected final Object infoLock = new Object(); /** * Track the ByteBuffers that we have handed out to readers. @@ -239,7 +239,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, * back to the namenode to get a new list of block locations, and is * capped at maxBlockAcquireFailures */ - private int failures = 0; + protected int failures = 0; /* XXX Use of CocurrentHashMap is temp fix. Need to fix * parallel accesses to DFSInputStream (through ptreads) properly */ @@ -476,7 +476,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, } /** Fetch a block from namenode and cache it */ - private void fetchBlockAt(long offset) throws IOException { + protected void fetchBlockAt(long
[49/50] hadoop git commit: HDFS-8033. Erasure coding: stateful (non-positional) read from files in striped layout. Contributed by Zhe Zhang.
HDFS-8033. Erasure coding: stateful (non-positional) read from files in striped layout. Contributed by Zhe Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/355f9cb2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/355f9cb2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/355f9cb2 Branch: refs/heads/HDFS-7285 Commit: 355f9cb259ccb7d456a612ed177986f2eaddc6fe Parents: cfd03ef Author: Zhe Zhang z...@apache.org Authored: Fri Apr 24 22:36:15 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 27 10:45:01 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 + .../org/apache/hadoop/hdfs/DFSInputStream.java | 55 ++-- .../hadoop/hdfs/DFSStripedInputStream.java | 311 ++- .../hadoop/hdfs/TestDFSStripedInputStream.java | 43 +++ .../apache/hadoop/hdfs/TestReadStripedFile.java | 110 ++- 5 files changed, 465 insertions(+), 57 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/355f9cb2/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index cf41a9b..e8db485 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -131,3 +131,6 @@ HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts (Jing Zhao via Zhe Zhang) + +HDFS-8033. Erasure coding: stateful (non-positional) read from files in +striped layout (Zhe Zhang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/355f9cb2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 705e0b7..7f267b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -95,34 +95,34 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, public static boolean tcpReadsDisabledForTesting = false; private long hedgedReadOpsLoopNumForTesting = 0; protected final DFSClient dfsClient; - private AtomicBoolean closed = new AtomicBoolean(false); - private final String src; - private final boolean verifyChecksum; + protected AtomicBoolean closed = new AtomicBoolean(false); + protected final String src; + protected final boolean verifyChecksum; // state by stateful read only: // (protected by lock on this) / private DatanodeInfo currentNode = null; - private LocatedBlock currentLocatedBlock = null; - private long pos = 0; - private long blockEnd = -1; + protected LocatedBlock currentLocatedBlock = null; + protected long pos = 0; + protected long blockEnd = -1; private BlockReader blockReader = null; // state shared by stateful and positional read: // (protected by lock on infoLock) - private LocatedBlocks locatedBlocks = null; + protected LocatedBlocks locatedBlocks = null; private long lastBlockBeingWrittenLength = 0; private FileEncryptionInfo fileEncryptionInfo = null; - private CachingStrategy cachingStrategy; + protected CachingStrategy cachingStrategy; - private final ReadStatistics readStatistics = new ReadStatistics(); + protected final ReadStatistics readStatistics = new ReadStatistics(); // lock for state shared between read and pread // Note: Never acquire a lock on this with this lock held to avoid deadlocks // (it's OK to acquire this lock when the lock on this is held) - private final Object infoLock = new Object(); + protected final Object infoLock = new Object(); /** * Track the ByteBuffers that we have handed out to readers. @@ -239,7 +239,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, * back to the namenode to get a new list of block locations, and is * capped at maxBlockAcquireFailures */ - private int failures = 0; + protected int failures = 0; /* XXX Use of CocurrentHashMap is temp fix. Need to fix * parallel accesses to DFSInputStream (through ptreads) properly */ @@ -476,7 +476,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, } /** Fetch a block from namenode and cache it */ - private void fetchBlockAt(long offset) throws IOException { + protected void fetchBlockAt(long
hadoop git commit: HDFS-8033. Erasure coding: stateful (non-positional) read from files in striped layout. Contributed by Zhe Zhang.
Repository: hadoop Updated Branches: refs/heads/HDFS-7285 b2ba6836b - 30e196354 HDFS-8033. Erasure coding: stateful (non-positional) read from files in striped layout. Contributed by Zhe Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/30e19635 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/30e19635 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/30e19635 Branch: refs/heads/HDFS-7285 Commit: 30e196354330031bc9d2e10ba3e61117e0a3aee5 Parents: b2ba683 Author: Zhe Zhang z...@apache.org Authored: Fri Apr 24 22:36:15 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Fri Apr 24 22:36:15 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 + .../org/apache/hadoop/hdfs/DFSInputStream.java | 55 ++-- .../hadoop/hdfs/DFSStripedInputStream.java | 311 ++- .../hadoop/hdfs/TestDFSStripedInputStream.java | 43 +++ .../apache/hadoop/hdfs/TestReadStripedFile.java | 110 ++- .../server/datanode/SimulatedFSDataset.java | 3 + 6 files changed, 468 insertions(+), 57 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/30e19635/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index cf41a9b..e8db485 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -131,3 +131,6 @@ HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause block id conflicts (Jing Zhao via Zhe Zhang) + +HDFS-8033. Erasure coding: stateful (non-positional) read from files in +striped layout (Zhe Zhang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/30e19635/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 705e0b7..7f267b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -95,34 +95,34 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, public static boolean tcpReadsDisabledForTesting = false; private long hedgedReadOpsLoopNumForTesting = 0; protected final DFSClient dfsClient; - private AtomicBoolean closed = new AtomicBoolean(false); - private final String src; - private final boolean verifyChecksum; + protected AtomicBoolean closed = new AtomicBoolean(false); + protected final String src; + protected final boolean verifyChecksum; // state by stateful read only: // (protected by lock on this) / private DatanodeInfo currentNode = null; - private LocatedBlock currentLocatedBlock = null; - private long pos = 0; - private long blockEnd = -1; + protected LocatedBlock currentLocatedBlock = null; + protected long pos = 0; + protected long blockEnd = -1; private BlockReader blockReader = null; // state shared by stateful and positional read: // (protected by lock on infoLock) - private LocatedBlocks locatedBlocks = null; + protected LocatedBlocks locatedBlocks = null; private long lastBlockBeingWrittenLength = 0; private FileEncryptionInfo fileEncryptionInfo = null; - private CachingStrategy cachingStrategy; + protected CachingStrategy cachingStrategy; - private final ReadStatistics readStatistics = new ReadStatistics(); + protected final ReadStatistics readStatistics = new ReadStatistics(); // lock for state shared between read and pread // Note: Never acquire a lock on this with this lock held to avoid deadlocks // (it's OK to acquire this lock when the lock on this is held) - private final Object infoLock = new Object(); + protected final Object infoLock = new Object(); /** * Track the ByteBuffers that we have handed out to readers. @@ -239,7 +239,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, * back to the namenode to get a new list of block locations, and is * capped at maxBlockAcquireFailures */ - private int failures = 0; + protected int failures = 0; /* XXX Use of CocurrentHashMap is temp fix. Need to fix * parallel accesses to DFSInputStream (through ptreads) properly */ @@ -476,7 +476,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, } /** Fetch