Repository: hadoop Updated Branches: refs/heads/trunk 652dd434d -> 3123895db
HDFS-12318. Fix IOException condition for openInfo in DFSInputStream. Contributed by legend. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3123895d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3123895d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3123895d Branch: refs/heads/trunk Commit: 3123895db0518552eb357093fddf0e8ff1a6cadf Parents: 652dd43 Author: John Zhuge <[email protected]> Authored: Thu Aug 24 00:26:37 2017 -0700 Committer: John Zhuge <[email protected]> Committed: Thu Aug 24 00:26:37 2017 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/DFSInputStream.java | 8 +++- .../apache/hadoop/hdfs/TestDFSInputStream.java | 41 ++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3123895d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 97d3de4..e3d7ade 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -182,6 +182,11 @@ public class DFSInputStream extends FSInputStream openInfo(false); } + @VisibleForTesting + public long getlastBlockBeingWrittenLengthForTesting() { + return lastBlockBeingWrittenLength; + } + /** * Grab the open-file info from namenode * @param refreshLocatedBlocks whether to re-fetch locatedblocks @@ -209,7 +214,8 @@ public class DFSInputStream extends FSInputStream } retriesForLastBlockLength--; } - if (retriesForLastBlockLength == 0) { + if (lastBlockBeingWrittenLength == -1 + && retriesForLastBlockLength == 0) { throw new IOException("Could not obtain the last block locations."); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3123895d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java index 2012258..eb4f124 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java @@ -33,6 +33,9 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.net.unix.DomainSocket; import org.apache.hadoop.net.unix.TemporarySocketDirectory; +import org.apache.hadoop.hdfs.client.impl.DfsClientConf; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Retry; + import org.junit.Assume; import org.junit.Test; @@ -135,4 +138,42 @@ public class TestDFSInputStream { cluster.shutdown(); } } + + @Test(timeout=60000) + public void testOpenInfo() throws IOException { + Configuration conf = new Configuration(); + conf.setInt(Retry.TIMES_GET_LAST_BLOCK_LENGTH_KEY, 0); + MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).build(); + cluster.waitActive(); + try { + DistributedFileSystem fs = cluster.getFileSystem(); + + int chunkSize = 512; + Random r = new Random(12345L); + byte[] data = new byte[chunkSize]; + r.nextBytes(data); + + Path file = new Path("/testfile"); + try(FSDataOutputStream fout = fs.create(file)) { + fout.write(data); + } + + DfsClientConf dcconf = new DfsClientConf(conf); + int retryTimesForGetLastBlockLength = + dcconf.getRetryTimesForGetLastBlockLength(); + assertEquals(0, retryTimesForGetLastBlockLength); + + try(DFSInputStream fin = fs.dfs.open("/testfile")) { + long flen = fin.getFileLength(); + assertEquals(chunkSize, flen); + + long lastBlockBeingWrittenLength = + fin.getlastBlockBeingWrittenLengthForTesting(); + assertEquals(0, lastBlockBeingWrittenLength); + } + } finally { + cluster.shutdown(); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
