Repository: hadoop Updated Branches: refs/heads/branch-2.8 01912d18d -> 90c1fac6e
HDFS-12318. Fix IOException condition for openInfo in DFSInputStream. Contributed by legend. (cherry picked from commit 3123895db0518552eb357093fddf0e8ff1a6cadf) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/90c1fac6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/90c1fac6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/90c1fac6 Branch: refs/heads/branch-2.8 Commit: 90c1fac6e30873cb515ca9304e5d2bd34dd18458 Parents: 01912d1 Author: John Zhuge <jzh...@apache.org> Authored: Thu Nov 9 09:22:30 2017 -0800 Committer: John Zhuge <jzh...@apache.org> Committed: Thu Nov 9 09:22:30 2017 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/DFSInputStream.java | 8 +++- .../apache/hadoop/hdfs/TestDFSInputStream.java | 41 ++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/90c1fac6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index b8705f5..972e536 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -270,6 +270,11 @@ public class DFSInputStream extends FSInputStream openInfo(false); } + @VisibleForTesting + public long getlastBlockBeingWrittenLengthForTesting() { + return lastBlockBeingWrittenLength; + } + /** * Grab the open-file info from namenode * @param refreshLocatedBlocks whether to re-fetch locatedblocks @@ -297,7 +302,8 @@ public class DFSInputStream extends FSInputStream } retriesForLastBlockLength--; } - if (retriesForLastBlockLength == 0) { + if (lastBlockBeingWrittenLength == -1 + && retriesForLastBlockLength == 0) { throw new IOException("Could not obtain the last block locations."); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/90c1fac6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java index 9d88384..5d0e730 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java @@ -33,6 +33,9 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.net.unix.DomainSocket; import org.apache.hadoop.net.unix.TemporarySocketDirectory; +import org.apache.hadoop.hdfs.client.impl.DfsClientConf; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Retry; + import org.junit.Assume; import org.junit.Test; @@ -136,4 +139,42 @@ public class TestDFSInputStream { cluster.shutdown(); } } + + @Test(timeout=60000) + public void testOpenInfo() throws IOException { + Configuration conf = new Configuration(); + conf.setInt(Retry.TIMES_GET_LAST_BLOCK_LENGTH_KEY, 0); + MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).build(); + cluster.waitActive(); + try { + DistributedFileSystem fs = cluster.getFileSystem(); + + int chunkSize = 512; + Random r = new Random(12345L); + byte[] data = new byte[chunkSize]; + r.nextBytes(data); + + Path file = new Path("/testfile"); + try(FSDataOutputStream fout = fs.create(file)) { + fout.write(data); + } + + DfsClientConf dcconf = new DfsClientConf(conf); + int retryTimesForGetLastBlockLength = + dcconf.getRetryTimesForGetLastBlockLength(); + assertEquals(0, retryTimesForGetLastBlockLength); + + try(DFSInputStream fin = fs.dfs.open("/testfile")) { + long flen = fin.getFileLength(); + assertEquals(chunkSize, flen); + + long lastBlockBeingWrittenLength = + fin.getlastBlockBeingWrittenLengthForTesting(); + assertEquals(0, lastBlockBeingWrittenLength); + } + } finally { + cluster.shutdown(); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
