hadoop git commit: HDFS-13926. ThreadLocal aggregations for FileSystem.Statistics are incorrect with striped reads. Contributed by Xiao Chen, Hrishikesh Gadre.

2018-10-09 Thread xiao 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 af85ce6ae -> b170de8be


HDFS-13926. ThreadLocal aggregations for FileSystem.Statistics are incorrect 
with striped reads.
Contributed by Xiao Chen, Hrishikesh Gadre.

Signed-off-by: Xiao Chen 
(cherry picked from commit 323b76bccfa153ef5ba52dc14876283d05618739)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b170de8b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b170de8b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b170de8b

Branch: refs/heads/branch-3.0
Commit: b170de8be5234015794d1e9eb09f5e69c7a2af25
Parents: af85ce6
Author: Hrishikesh Gadre 
Authored: Tue Oct 9 16:42:22 2018 -0700
Committer: Xiao Chen 
Committed: Tue Oct 9 19:54:56 2018 -0700

--
 .../org/apache/hadoop/hdfs/DFSInputStream.java  |  5 ++
 .../hadoop/hdfs/DFSStripedInputStream.java  | 20 ++
 .../org/apache/hadoop/hdfs/ReaderStrategy.java  |  7 ---
 .../org/apache/hadoop/hdfs/StripeReader.java| 23 ---
 .../apache/hadoop/hdfs/util/IOUtilsClient.java  | 10 ++-
 .../hadoop/hdfs/util/StripedBlockUtil.java  | 65 ++--
 .../erasurecode/ErasureCodingWorker.java|  3 +-
 .../erasurecode/StripedBlockReader.java | 14 +++--
 .../datanode/erasurecode/StripedReader.java | 17 ++---
 .../erasurecode/StripedReconstructor.java   |  3 +-
 .../TestDistributedFileSystemWithECFile.java| 38 
 11 files changed, 167 insertions(+), 38 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b170de8b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index ae24572..827db47 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -89,6 +89,8 @@ import com.google.common.annotations.VisibleForTesting;
 
 import javax.annotation.Nonnull;
 
+import static org.apache.hadoop.hdfs.util.IOUtilsClient.updateReadStatistics;
+
 /
  * DFSInputStream provides bytes from a named file.  It handles
  * negotiation of the namenode and various datanodes as necessary.
@@ -768,6 +770,9 @@ public class DFSInputStream extends FSInputStream
 // got a EOS from reader though we expect more data on it.
 throw new IOException("Unexpected EOS from the reader");
   }
+  updateReadStatistics(readStatistics, result, blockReader);
+  dfsClient.updateFileSystemReadStats(blockReader.getNetworkDistance(),
+  result);
   return result;
 } catch (ChecksumException ce) {
   throw ce;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b170de8b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 190ba8e..9ec3e0b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -53,6 +53,8 @@ import java.util.Collection;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ThreadPoolExecutor;
 
+import static org.apache.hadoop.hdfs.util.IOUtilsClient.updateReadStatistics;
+
 /**
  * DFSStripedInputStream reads from striped block groups.
  */
@@ -327,6 +329,24 @@ public class DFSStripedInputStream extends DFSInputStream {
   }
 
   /**
+   * Update read statistics. Note that this has to be done on the thread that
+   * initiates the read, rather than inside each async thread, for
+   * {@link org.apache.hadoop.fs.FileSystem.Statistics} to work correctly with
+   * its ThreadLocal.
+   *
+   * @param stats striped read stats
+   */
+  void updateReadStats(final StripedBlockUtil.BlockReadStats stats) {
+if (stats == null) {
+  return;
+}
+updateReadStatistics(readStatistics, stats.getBytesRead(),
+stats.isShortCircuit(), stats.getNetworkDistance());
+dfsClient.updateFileSystemReadStats(stats.getNetworkDistance(),
+stats.getBytesRead());
+  }
+
+

hadoop git commit: HDFS-13926. ThreadLocal aggregations for FileSystem.Statistics are incorrect with striped reads. Contributed by Xiao Chen, Hrishikesh Gadre.

2018-10-09 Thread xiao 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.1 3968ce107 -> 323b76bcc


HDFS-13926. ThreadLocal aggregations for FileSystem.Statistics are incorrect 
with striped reads.
Contributed by Xiao Chen, Hrishikesh Gadre.

Signed-off-by: Xiao Chen 


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/323b76bc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/323b76bc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/323b76bc

Branch: refs/heads/branch-3.1
Commit: 323b76bccfa153ef5ba52dc14876283d05618739
Parents: 3968ce1
Author: Hrishikesh Gadre 
Authored: Tue Oct 9 16:42:22 2018 -0700
Committer: Xiao Chen 
Committed: Tue Oct 9 19:54:34 2018 -0700

--
 .../org/apache/hadoop/hdfs/DFSInputStream.java  |  5 ++
 .../hadoop/hdfs/DFSStripedInputStream.java  | 20 ++
 .../org/apache/hadoop/hdfs/ReaderStrategy.java  |  7 ---
 .../org/apache/hadoop/hdfs/StripeReader.java| 23 ---
 .../apache/hadoop/hdfs/util/IOUtilsClient.java  | 10 ++-
 .../hadoop/hdfs/util/StripedBlockUtil.java  | 65 ++--
 .../erasurecode/ErasureCodingWorker.java|  3 +-
 .../erasurecode/StripedBlockReader.java | 14 +++--
 .../datanode/erasurecode/StripedReader.java | 17 ++---
 .../erasurecode/StripedReconstructor.java   |  3 +-
 .../TestDistributedFileSystemWithECFile.java| 38 
 11 files changed, 167 insertions(+), 38 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/323b76bc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 75eb2ea..98c2c9e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -89,6 +89,8 @@ import com.google.common.annotations.VisibleForTesting;
 
 import javax.annotation.Nonnull;
 
+import static org.apache.hadoop.hdfs.util.IOUtilsClient.updateReadStatistics;
+
 /
  * DFSInputStream provides bytes from a named file.  It handles
  * negotiation of the namenode and various datanodes as necessary.
@@ -768,6 +770,9 @@ public class DFSInputStream extends FSInputStream
 // got a EOS from reader though we expect more data on it.
 throw new IOException("Unexpected EOS from the reader");
   }
+  updateReadStatistics(readStatistics, result, blockReader);
+  dfsClient.updateFileSystemReadStats(blockReader.getNetworkDistance(),
+  result);
   return result;
 } catch (ChecksumException ce) {
   throw ce;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/323b76bc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 190ba8e..9ec3e0b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -53,6 +53,8 @@ import java.util.Collection;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ThreadPoolExecutor;
 
+import static org.apache.hadoop.hdfs.util.IOUtilsClient.updateReadStatistics;
+
 /**
  * DFSStripedInputStream reads from striped block groups.
  */
@@ -327,6 +329,24 @@ public class DFSStripedInputStream extends DFSInputStream {
   }
 
   /**
+   * Update read statistics. Note that this has to be done on the thread that
+   * initiates the read, rather than inside each async thread, for
+   * {@link org.apache.hadoop.fs.FileSystem.Statistics} to work correctly with
+   * its ThreadLocal.
+   *
+   * @param stats striped read stats
+   */
+  void updateReadStats(final StripedBlockUtil.BlockReadStats stats) {
+if (stats == null) {
+  return;
+}
+updateReadStatistics(readStatistics, stats.getBytesRead(),
+stats.isShortCircuit(), stats.getNetworkDistance());
+dfsClient.updateFileSystemReadStats(stats.getNetworkDistance(),
+stats.getBytesRead());
+  }
+
+  /**
* Seek to a new arbitrary location.
*/
   @Override

hadoop git commit: HDFS-13926. ThreadLocal aggregations for FileSystem.Statistics are incorrect with striped reads. Contributed by Xiao Chen, Hrishikesh Gadre.

2018-10-08 Thread xiao 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.2 b6698e2a8 -> a99658cd8


HDFS-13926. ThreadLocal aggregations for FileSystem.Statistics are incorrect 
with striped reads.
Contributed by Xiao Chen, Hrishikesh Gadre.

Signed-off-by: Xiao Chen 
(cherry picked from commit 08bb6c49a5aec32b7d9f29238560f947420405d6)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a99658cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a99658cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a99658cd

Branch: refs/heads/branch-3.2
Commit: a99658cd8504edfe7b80f979eb25795c16726dcd
Parents: b6698e2
Author: Hrishikesh Gadre 
Authored: Mon Oct 8 20:30:53 2018 -0700
Committer: Xiao Chen 
Committed: Mon Oct 8 20:39:48 2018 -0700

--
 .../org/apache/hadoop/hdfs/DFSInputStream.java  |  8 +++
 .../hadoop/hdfs/DFSStripedInputStream.java  | 22 +++
 .../org/apache/hadoop/hdfs/ReaderStrategy.java  | 15 -
 .../org/apache/hadoop/hdfs/StripeReader.java| 23 ---
 .../apache/hadoop/hdfs/util/IOUtilsClient.java  | 10 ++-
 .../hadoop/hdfs/util/StripedBlockUtil.java  | 65 ++--
 .../erasurecode/ErasureCodingWorker.java|  3 +-
 .../erasurecode/StripedBlockReader.java | 14 +++--
 .../datanode/erasurecode/StripedReader.java | 17 ++---
 .../erasurecode/StripedReconstructor.java   |  3 +-
 .../TestDistributedFileSystemWithECFile.java| 44 +
 11 files changed, 178 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a99658cd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index e5640d2..52ed1d4 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -90,6 +90,8 @@ import com.google.common.annotations.VisibleForTesting;
 
 import javax.annotation.Nonnull;
 
+import static org.apache.hadoop.hdfs.util.IOUtilsClient.updateReadStatistics;
+
 /
  * DFSInputStream provides bytes from a named file.  It handles
  * negotiation of the namenode and various datanodes as necessary.
@@ -769,6 +771,12 @@ public class DFSInputStream extends FSInputStream
 // got a EOS from reader though we expect more data on it.
 throw new IOException("Unexpected EOS from the reader");
   }
+  updateReadStatistics(readStatistics, result, blockReader);
+  dfsClient.updateFileSystemReadStats(blockReader.getNetworkDistance(),
+  result);
+  if (readStatistics.getBlockType() == BlockType.STRIPED) {
+dfsClient.updateFileSystemECReadStats(result);
+  }
   return result;
 } catch (ChecksumException ce) {
   throw ce;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a99658cd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 5557a50..3f688d4 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -54,6 +54,8 @@ import java.util.Collection;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ThreadPoolExecutor;
 
+import static org.apache.hadoop.hdfs.util.IOUtilsClient.updateReadStatistics;
+
 /**
  * DFSStripedInputStream reads from striped block groups.
  */
@@ -329,6 +331,26 @@ public class DFSStripedInputStream extends DFSInputStream {
   }
 
   /**
+   * Update read statistics. Note that this has to be done on the thread that
+   * initiates the read, rather than inside each async thread, for
+   * {@link org.apache.hadoop.fs.FileSystem.Statistics} to work correctly with
+   * its ThreadLocal.
+   *
+   * @param stats striped read stats
+   */
+  void updateReadStats(final StripedBlockUtil.BlockReadStats stats) {
+if (stats == null) {
+  return;
+}
+updateReadStatistics(readStatistics, stats.getBytesRead(),
+

hadoop git commit: HDFS-13926. ThreadLocal aggregations for FileSystem.Statistics are incorrect with striped reads. Contributed by Xiao Chen, Hrishikesh Gadre.

2018-10-08 Thread xiao 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 1043795f7 -> 08bb6c49a


HDFS-13926. ThreadLocal aggregations for FileSystem.Statistics are incorrect 
with striped reads.
Contributed by Xiao Chen, Hrishikesh Gadre.

Signed-off-by: Xiao Chen 


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/08bb6c49
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/08bb6c49
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/08bb6c49

Branch: refs/heads/trunk
Commit: 08bb6c49a5aec32b7d9f29238560f947420405d6
Parents: 1043795
Author: Hrishikesh Gadre 
Authored: Mon Oct 8 20:30:53 2018 -0700
Committer: Xiao Chen 
Committed: Mon Oct 8 20:31:57 2018 -0700

--
 .../org/apache/hadoop/hdfs/DFSInputStream.java  |  8 +++
 .../hadoop/hdfs/DFSStripedInputStream.java  | 22 +++
 .../org/apache/hadoop/hdfs/ReaderStrategy.java  | 15 -
 .../org/apache/hadoop/hdfs/StripeReader.java| 23 ---
 .../apache/hadoop/hdfs/util/IOUtilsClient.java  | 10 ++-
 .../hadoop/hdfs/util/StripedBlockUtil.java  | 65 ++--
 .../erasurecode/ErasureCodingWorker.java|  3 +-
 .../erasurecode/StripedBlockReader.java | 14 +++--
 .../datanode/erasurecode/StripedReader.java | 17 ++---
 .../erasurecode/StripedReconstructor.java   |  3 +-
 .../TestDistributedFileSystemWithECFile.java| 44 +
 11 files changed, 178 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb6c49/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index e5640d2..52ed1d4 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -90,6 +90,8 @@ import com.google.common.annotations.VisibleForTesting;
 
 import javax.annotation.Nonnull;
 
+import static org.apache.hadoop.hdfs.util.IOUtilsClient.updateReadStatistics;
+
 /
  * DFSInputStream provides bytes from a named file.  It handles
  * negotiation of the namenode and various datanodes as necessary.
@@ -769,6 +771,12 @@ public class DFSInputStream extends FSInputStream
 // got a EOS from reader though we expect more data on it.
 throw new IOException("Unexpected EOS from the reader");
   }
+  updateReadStatistics(readStatistics, result, blockReader);
+  dfsClient.updateFileSystemReadStats(blockReader.getNetworkDistance(),
+  result);
+  if (readStatistics.getBlockType() == BlockType.STRIPED) {
+dfsClient.updateFileSystemECReadStats(result);
+  }
   return result;
 } catch (ChecksumException ce) {
   throw ce;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb6c49/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 5557a50..3f688d4 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -54,6 +54,8 @@ import java.util.Collection;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ThreadPoolExecutor;
 
+import static org.apache.hadoop.hdfs.util.IOUtilsClient.updateReadStatistics;
+
 /**
  * DFSStripedInputStream reads from striped block groups.
  */
@@ -329,6 +331,26 @@ public class DFSStripedInputStream extends DFSInputStream {
   }
 
   /**
+   * Update read statistics. Note that this has to be done on the thread that
+   * initiates the read, rather than inside each async thread, for
+   * {@link org.apache.hadoop.fs.FileSystem.Statistics} to work correctly with
+   * its ThreadLocal.
+   *
+   * @param stats striped read stats
+   */
+  void updateReadStats(final StripedBlockUtil.BlockReadStats stats) {
+if (stats == null) {
+  return;
+}
+updateReadStatistics(readStatistics, stats.getBytesRead(),
+stats.isShortCircuit(), stats.getNetworkDistance());
+