hadoop git commit: HDFS-12318. Fix IOException condition for openInfo in DFSInputStream. Contributed by legend.

2017-11-09 Thread jzhuge 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.9 26110ec33 -> 120d86896


HDFS-12318. Fix IOException condition for openInfo in DFSInputStream. 
Contributed by legend.

(cherry picked from commit 3123895db0518552eb357093fddf0e8ff1a6cadf)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/120d8689
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/120d8689
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/120d8689

Branch: refs/heads/branch-2.9
Commit: 120d8689638b3197d46487a6d5d50e3f5b2c0273
Parents: 26110ec
Author: John Zhuge 
Authored: Thu Nov 9 09:18:53 2017 -0800
Committer: John Zhuge 
Committed: Thu Nov 9 09:18:53 2017 -0800

--
 .../org/apache/hadoop/hdfs/DFSInputStream.java  |  8 +++-
 .../apache/hadoop/hdfs/TestDFSInputStream.java  | 41 
 2 files changed, 48 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/120d8689/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index eb566b3..3e50d33 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -272,6 +272,11 @@ public class DFSInputStream extends FSInputStream
 openInfo(false);
   }
 
+  @VisibleForTesting
+  public long getlastBlockBeingWrittenLengthForTesting() {
+return lastBlockBeingWrittenLength;
+  }
+
   /**
* Grab the open-file info from namenode
* @param refreshLocatedBlocks whether to re-fetch locatedblocks
@@ -299,7 +304,8 @@ public class DFSInputStream extends FSInputStream
 }
 retriesForLastBlockLength--;
   }
-  if (retriesForLastBlockLength == 0) {
+  if (lastBlockBeingWrittenLength == -1
+  && retriesForLastBlockLength == 0) {
 throw new IOException("Could not obtain the last block locations.");
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/120d8689/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
index 9d88384..5d0e730 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
@@ -33,6 +33,9 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.net.unix.DomainSocket;
 import org.apache.hadoop.net.unix.TemporarySocketDirectory;
+import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Retry;
+
 import org.junit.Assume;
 import org.junit.Test;
 
@@ -136,4 +139,42 @@ public class TestDFSInputStream {
   cluster.shutdown();
 }
   }
+
+  @Test(timeout=6)
+  public void testOpenInfo() throws IOException {
+Configuration conf = new Configuration();
+conf.setInt(Retry.TIMES_GET_LAST_BLOCK_LENGTH_KEY, 0);
+MiniDFSCluster cluster =
+new MiniDFSCluster.Builder(conf).build();
+cluster.waitActive();
+try {
+  DistributedFileSystem fs = cluster.getFileSystem();
+
+  int chunkSize = 512;
+  Random r = new Random(12345L);
+  byte[] data = new byte[chunkSize];
+  r.nextBytes(data);
+
+  Path file = new Path("/testfile");
+  try(FSDataOutputStream fout = fs.create(file)) {
+fout.write(data);
+  }
+
+  DfsClientConf dcconf = new DfsClientConf(conf);
+  int retryTimesForGetLastBlockLength =
+  dcconf.getRetryTimesForGetLastBlockLength();
+  assertEquals(0, retryTimesForGetLastBlockLength);
+
+  try(DFSInputStream fin = fs.dfs.open("/testfile")) {
+long flen = fin.getFileLength();
+assertEquals(chunkSize, flen);
+
+long lastBlockBeingWrittenLength =
+fin.getlastBlockBeingWrittenLengthForTesting();
+assertEquals(0, lastBlockBeingWrittenLength);
+  }
+} finally {
+  cluster.shutdown();
+}
+  }
 }


-
To unsubscribe, e-mail:

hadoop git commit: HDFS-12318. Fix IOException condition for openInfo in DFSInputStream. Contributed by legend.

2017-11-09 Thread jzhuge 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 01912d18d -> 90c1fac6e


HDFS-12318. Fix IOException condition for openInfo in DFSInputStream. 
Contributed by legend.

(cherry picked from commit 3123895db0518552eb357093fddf0e8ff1a6cadf)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/90c1fac6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/90c1fac6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/90c1fac6

Branch: refs/heads/branch-2.8
Commit: 90c1fac6e30873cb515ca9304e5d2bd34dd18458
Parents: 01912d1
Author: John Zhuge 
Authored: Thu Nov 9 09:22:30 2017 -0800
Committer: John Zhuge 
Committed: Thu Nov 9 09:22:30 2017 -0800

--
 .../org/apache/hadoop/hdfs/DFSInputStream.java  |  8 +++-
 .../apache/hadoop/hdfs/TestDFSInputStream.java  | 41 
 2 files changed, 48 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/90c1fac6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index b8705f5..972e536 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -270,6 +270,11 @@ public class DFSInputStream extends FSInputStream
 openInfo(false);
   }
 
+  @VisibleForTesting
+  public long getlastBlockBeingWrittenLengthForTesting() {
+return lastBlockBeingWrittenLength;
+  }
+
   /**
* Grab the open-file info from namenode
* @param refreshLocatedBlocks whether to re-fetch locatedblocks
@@ -297,7 +302,8 @@ public class DFSInputStream extends FSInputStream
 }
 retriesForLastBlockLength--;
   }
-  if (retriesForLastBlockLength == 0) {
+  if (lastBlockBeingWrittenLength == -1
+  && retriesForLastBlockLength == 0) {
 throw new IOException("Could not obtain the last block locations.");
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/90c1fac6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
index 9d88384..5d0e730 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
@@ -33,6 +33,9 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.net.unix.DomainSocket;
 import org.apache.hadoop.net.unix.TemporarySocketDirectory;
+import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Retry;
+
 import org.junit.Assume;
 import org.junit.Test;
 
@@ -136,4 +139,42 @@ public class TestDFSInputStream {
   cluster.shutdown();
 }
   }
+
+  @Test(timeout=6)
+  public void testOpenInfo() throws IOException {
+Configuration conf = new Configuration();
+conf.setInt(Retry.TIMES_GET_LAST_BLOCK_LENGTH_KEY, 0);
+MiniDFSCluster cluster =
+new MiniDFSCluster.Builder(conf).build();
+cluster.waitActive();
+try {
+  DistributedFileSystem fs = cluster.getFileSystem();
+
+  int chunkSize = 512;
+  Random r = new Random(12345L);
+  byte[] data = new byte[chunkSize];
+  r.nextBytes(data);
+
+  Path file = new Path("/testfile");
+  try(FSDataOutputStream fout = fs.create(file)) {
+fout.write(data);
+  }
+
+  DfsClientConf dcconf = new DfsClientConf(conf);
+  int retryTimesForGetLastBlockLength =
+  dcconf.getRetryTimesForGetLastBlockLength();
+  assertEquals(0, retryTimesForGetLastBlockLength);
+
+  try(DFSInputStream fin = fs.dfs.open("/testfile")) {
+long flen = fin.getFileLength();
+assertEquals(chunkSize, flen);
+
+long lastBlockBeingWrittenLength =
+fin.getlastBlockBeingWrittenLengthForTesting();
+assertEquals(0, lastBlockBeingWrittenLength);
+  }
+} finally {
+  cluster.shutdown();
+}
+  }
 }


-
To unsubscribe, e-mail:

hadoop git commit: HDFS-12318. Fix IOException condition for openInfo in DFSInputStream. Contributed by legend.

2017-11-09 Thread jzhuge 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 5991c218a -> d946307fa


HDFS-12318. Fix IOException condition for openInfo in DFSInputStream. 
Contributed by legend.

(cherry picked from commit 3123895db0518552eb357093fddf0e8ff1a6cadf)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d946307f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d946307f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d946307f

Branch: refs/heads/branch-2
Commit: d946307faa87946976fe2775b18dc1e3963b762a
Parents: 5991c21
Author: John Zhuge 
Authored: Thu Nov 9 08:58:03 2017 -0800
Committer: John Zhuge 
Committed: Thu Nov 9 08:58:03 2017 -0800

--
 .../org/apache/hadoop/hdfs/DFSInputStream.java  |  8 +++-
 .../apache/hadoop/hdfs/TestDFSInputStream.java  | 41 
 2 files changed, 48 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d946307f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index eb566b3..3e50d33 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -272,6 +272,11 @@ public class DFSInputStream extends FSInputStream
 openInfo(false);
   }
 
+  @VisibleForTesting
+  public long getlastBlockBeingWrittenLengthForTesting() {
+return lastBlockBeingWrittenLength;
+  }
+
   /**
* Grab the open-file info from namenode
* @param refreshLocatedBlocks whether to re-fetch locatedblocks
@@ -299,7 +304,8 @@ public class DFSInputStream extends FSInputStream
 }
 retriesForLastBlockLength--;
   }
-  if (retriesForLastBlockLength == 0) {
+  if (lastBlockBeingWrittenLength == -1
+  && retriesForLastBlockLength == 0) {
 throw new IOException("Could not obtain the last block locations.");
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d946307f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
index 9d88384..5d0e730 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
@@ -33,6 +33,9 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.net.unix.DomainSocket;
 import org.apache.hadoop.net.unix.TemporarySocketDirectory;
+import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Retry;
+
 import org.junit.Assume;
 import org.junit.Test;
 
@@ -136,4 +139,42 @@ public class TestDFSInputStream {
   cluster.shutdown();
 }
   }
+
+  @Test(timeout=6)
+  public void testOpenInfo() throws IOException {
+Configuration conf = new Configuration();
+conf.setInt(Retry.TIMES_GET_LAST_BLOCK_LENGTH_KEY, 0);
+MiniDFSCluster cluster =
+new MiniDFSCluster.Builder(conf).build();
+cluster.waitActive();
+try {
+  DistributedFileSystem fs = cluster.getFileSystem();
+
+  int chunkSize = 512;
+  Random r = new Random(12345L);
+  byte[] data = new byte[chunkSize];
+  r.nextBytes(data);
+
+  Path file = new Path("/testfile");
+  try(FSDataOutputStream fout = fs.create(file)) {
+fout.write(data);
+  }
+
+  DfsClientConf dcconf = new DfsClientConf(conf);
+  int retryTimesForGetLastBlockLength =
+  dcconf.getRetryTimesForGetLastBlockLength();
+  assertEquals(0, retryTimesForGetLastBlockLength);
+
+  try(DFSInputStream fin = fs.dfs.open("/testfile")) {
+long flen = fin.getFileLength();
+assertEquals(chunkSize, flen);
+
+long lastBlockBeingWrittenLength =
+fin.getlastBlockBeingWrittenLengthForTesting();
+assertEquals(0, lastBlockBeingWrittenLength);
+  }
+} finally {
+  cluster.shutdown();
+}
+  }
 }


-
To unsubscribe, e-mail:

hadoop git commit: HDFS-12318. Fix IOException condition for openInfo in DFSInputStream. Contributed by legend.

2017-08-24 Thread jzhuge 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 652dd434d -> 3123895db


HDFS-12318. Fix IOException condition for openInfo in DFSInputStream. 
Contributed by legend.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3123895d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3123895d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3123895d

Branch: refs/heads/trunk
Commit: 3123895db0518552eb357093fddf0e8ff1a6cadf
Parents: 652dd43
Author: John Zhuge 
Authored: Thu Aug 24 00:26:37 2017 -0700
Committer: John Zhuge 
Committed: Thu Aug 24 00:26:37 2017 -0700

--
 .../org/apache/hadoop/hdfs/DFSInputStream.java  |  8 +++-
 .../apache/hadoop/hdfs/TestDFSInputStream.java  | 41 
 2 files changed, 48 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3123895d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 97d3de4..e3d7ade 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -182,6 +182,11 @@ public class DFSInputStream extends FSInputStream
 openInfo(false);
   }
 
+  @VisibleForTesting
+  public long getlastBlockBeingWrittenLengthForTesting() {
+return lastBlockBeingWrittenLength;
+  }
+
   /**
* Grab the open-file info from namenode
* @param refreshLocatedBlocks whether to re-fetch locatedblocks
@@ -209,7 +214,8 @@ public class DFSInputStream extends FSInputStream
 }
 retriesForLastBlockLength--;
   }
-  if (retriesForLastBlockLength == 0) {
+  if (lastBlockBeingWrittenLength == -1
+  && retriesForLastBlockLength == 0) {
 throw new IOException("Could not obtain the last block locations.");
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3123895d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
index 2012258..eb4f124 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
@@ -33,6 +33,9 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.net.unix.DomainSocket;
 import org.apache.hadoop.net.unix.TemporarySocketDirectory;
+import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Retry;
+
 import org.junit.Assume;
 import org.junit.Test;
 
@@ -135,4 +138,42 @@ public class TestDFSInputStream {
   cluster.shutdown();
 }
   }
+
+  @Test(timeout=6)
+  public void testOpenInfo() throws IOException {
+Configuration conf = new Configuration();
+conf.setInt(Retry.TIMES_GET_LAST_BLOCK_LENGTH_KEY, 0);
+MiniDFSCluster cluster =
+new MiniDFSCluster.Builder(conf).build();
+cluster.waitActive();
+try {
+  DistributedFileSystem fs = cluster.getFileSystem();
+
+  int chunkSize = 512;
+  Random r = new Random(12345L);
+  byte[] data = new byte[chunkSize];
+  r.nextBytes(data);
+
+  Path file = new Path("/testfile");
+  try(FSDataOutputStream fout = fs.create(file)) {
+fout.write(data);
+  }
+
+  DfsClientConf dcconf = new DfsClientConf(conf);
+  int retryTimesForGetLastBlockLength =
+  dcconf.getRetryTimesForGetLastBlockLength();
+  assertEquals(0, retryTimesForGetLastBlockLength);
+
+  try(DFSInputStream fin = fs.dfs.open("/testfile")) {
+long flen = fin.getFileLength();
+assertEquals(chunkSize, flen);
+
+long lastBlockBeingWrittenLength =
+fin.getlastBlockBeingWrittenLengthForTesting();
+assertEquals(0, lastBlockBeingWrittenLength);
+  }
+} finally {
+  cluster.shutdown();
+}
+  }
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: