[10/29] hadoop git commit: HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by Sammi Chen and Andrew Wang

2016-12-17 Thread asuresh
HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by 
Sammi Chen and Andrew Wang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f14f6d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f14f6d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f14f6d0

Branch: refs/heads/YARN-5085
Commit: 1f14f6d038aecad55a5398c6fa4137c9d2f44729
Parents: ada876c
Author: Kai Zheng 
Authored: Wed Dec 14 14:50:50 2016 +0800
Committer: Kai Zheng 
Committed: Wed Dec 14 14:50:50 2016 +0800

--
 .../erasurecode/StripedBlockReader.java |   1 +
 .../erasurecode/StripedBlockReconstructor.java  |   6 +-
 .../erasurecode/StripedBlockWriter.java |   1 +
 .../datanode/erasurecode/StripedReader.java |   4 +
 .../erasurecode/StripedReconstructor.java   |  21 +++
 .../datanode/erasurecode/StripedWriter.java |   4 +
 .../datanode/metrics/DataNodeMetrics.java   |  18 ++-
 .../apache/hadoop/hdfs/StripedFileTestUtil.java |  24 +++
 .../TestDataNodeErasureCodingMetrics.java   | 149 +--
 9 files changed, 147 insertions(+), 81 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
index a27de9b..0f7c5c7 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
@@ -187,6 +187,7 @@ class StripedBlockReader {
 break;
   }
   n += nread;
+  stripedReader.getReconstructor().incrBytesRead(nread);
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
index a8e9d30..5554d68 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
 
 /**
  * StripedBlockReconstructor reconstruct one or more missed striped block in
@@ -66,7 +67,10 @@ class StripedBlockReconstructor extends StripedReconstructor
   getDatanode().getMetrics().incrECFailedReconstructionTasks();
 } finally {
   getDatanode().decrementXmitsInProgress();
-  getDatanode().getMetrics().incrECReconstructionTasks();
+  final DataNodeMetrics metrics = getDatanode().getMetrics();
+  metrics.incrECReconstructionTasks();
+  metrics.incrECReconstructionBytesRead(getBytesRead());
+  metrics.incrECReconstructionBytesWritten(getBytesWritten());
   getStripedReader().close();
   stripedWriter.close();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
index 592be45..d999202 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
@@ -196,6 +196,7 @@ class StripedBlockWriter {
   packet.writeTo(targetOutputStream);
 
   blockOffset4Target += toWrite;
+  

[34/50] [abbrv] hadoop git commit: HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by Sammi Chen and Andrew Wang

2016-12-15 Thread rakeshr
HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by 
Sammi Chen and Andrew Wang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f14f6d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f14f6d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f14f6d0

Branch: refs/heads/HDFS-10285
Commit: 1f14f6d038aecad55a5398c6fa4137c9d2f44729
Parents: ada876c
Author: Kai Zheng 
Authored: Wed Dec 14 14:50:50 2016 +0800
Committer: Kai Zheng 
Committed: Wed Dec 14 14:50:50 2016 +0800

--
 .../erasurecode/StripedBlockReader.java |   1 +
 .../erasurecode/StripedBlockReconstructor.java  |   6 +-
 .../erasurecode/StripedBlockWriter.java |   1 +
 .../datanode/erasurecode/StripedReader.java |   4 +
 .../erasurecode/StripedReconstructor.java   |  21 +++
 .../datanode/erasurecode/StripedWriter.java |   4 +
 .../datanode/metrics/DataNodeMetrics.java   |  18 ++-
 .../apache/hadoop/hdfs/StripedFileTestUtil.java |  24 +++
 .../TestDataNodeErasureCodingMetrics.java   | 149 +--
 9 files changed, 147 insertions(+), 81 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
index a27de9b..0f7c5c7 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
@@ -187,6 +187,7 @@ class StripedBlockReader {
 break;
   }
   n += nread;
+  stripedReader.getReconstructor().incrBytesRead(nread);
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
index a8e9d30..5554d68 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
 
 /**
  * StripedBlockReconstructor reconstruct one or more missed striped block in
@@ -66,7 +67,10 @@ class StripedBlockReconstructor extends StripedReconstructor
   getDatanode().getMetrics().incrECFailedReconstructionTasks();
 } finally {
   getDatanode().decrementXmitsInProgress();
-  getDatanode().getMetrics().incrECReconstructionTasks();
+  final DataNodeMetrics metrics = getDatanode().getMetrics();
+  metrics.incrECReconstructionTasks();
+  metrics.incrECReconstructionBytesRead(getBytesRead());
+  metrics.incrECReconstructionBytesWritten(getBytesWritten());
   getStripedReader().close();
   stripedWriter.close();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
index 592be45..d999202 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
@@ -196,6 +196,7 @@ class StripedBlockWriter {
   packet.writeTo(targetOutputStream);
 
   blockOffset4Target += toWrite;
+  

[5/9] hadoop git commit: HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by Sammi Chen and Andrew Wang

2016-12-14 Thread xgong
HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by 
Sammi Chen and Andrew Wang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f14f6d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f14f6d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f14f6d0

Branch: refs/heads/YARN-5734
Commit: 1f14f6d038aecad55a5398c6fa4137c9d2f44729
Parents: ada876c
Author: Kai Zheng 
Authored: Wed Dec 14 14:50:50 2016 +0800
Committer: Kai Zheng 
Committed: Wed Dec 14 14:50:50 2016 +0800

--
 .../erasurecode/StripedBlockReader.java |   1 +
 .../erasurecode/StripedBlockReconstructor.java  |   6 +-
 .../erasurecode/StripedBlockWriter.java |   1 +
 .../datanode/erasurecode/StripedReader.java |   4 +
 .../erasurecode/StripedReconstructor.java   |  21 +++
 .../datanode/erasurecode/StripedWriter.java |   4 +
 .../datanode/metrics/DataNodeMetrics.java   |  18 ++-
 .../apache/hadoop/hdfs/StripedFileTestUtil.java |  24 +++
 .../TestDataNodeErasureCodingMetrics.java   | 149 +--
 9 files changed, 147 insertions(+), 81 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
index a27de9b..0f7c5c7 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
@@ -187,6 +187,7 @@ class StripedBlockReader {
 break;
   }
   n += nread;
+  stripedReader.getReconstructor().incrBytesRead(nread);
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
index a8e9d30..5554d68 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
 
 /**
  * StripedBlockReconstructor reconstruct one or more missed striped block in
@@ -66,7 +67,10 @@ class StripedBlockReconstructor extends StripedReconstructor
   getDatanode().getMetrics().incrECFailedReconstructionTasks();
 } finally {
   getDatanode().decrementXmitsInProgress();
-  getDatanode().getMetrics().incrECReconstructionTasks();
+  final DataNodeMetrics metrics = getDatanode().getMetrics();
+  metrics.incrECReconstructionTasks();
+  metrics.incrECReconstructionBytesRead(getBytesRead());
+  metrics.incrECReconstructionBytesWritten(getBytesWritten());
   getStripedReader().close();
   stripedWriter.close();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
index 592be45..d999202 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
@@ -196,6 +196,7 @@ class StripedBlockWriter {
   packet.writeTo(targetOutputStream);
 
   blockOffset4Target += toWrite;
+  

hadoop git commit: HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by Sammi Chen and Andrew Wang

2016-12-13 Thread drankye
Repository: hadoop
Updated Branches:
  refs/heads/trunk ada876cd1 -> 1f14f6d03


HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by 
Sammi Chen and Andrew Wang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f14f6d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f14f6d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f14f6d0

Branch: refs/heads/trunk
Commit: 1f14f6d038aecad55a5398c6fa4137c9d2f44729
Parents: ada876c
Author: Kai Zheng 
Authored: Wed Dec 14 14:50:50 2016 +0800
Committer: Kai Zheng 
Committed: Wed Dec 14 14:50:50 2016 +0800

--
 .../erasurecode/StripedBlockReader.java |   1 +
 .../erasurecode/StripedBlockReconstructor.java  |   6 +-
 .../erasurecode/StripedBlockWriter.java |   1 +
 .../datanode/erasurecode/StripedReader.java |   4 +
 .../erasurecode/StripedReconstructor.java   |  21 +++
 .../datanode/erasurecode/StripedWriter.java |   4 +
 .../datanode/metrics/DataNodeMetrics.java   |  18 ++-
 .../apache/hadoop/hdfs/StripedFileTestUtil.java |  24 +++
 .../TestDataNodeErasureCodingMetrics.java   | 149 +--
 9 files changed, 147 insertions(+), 81 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
index a27de9b..0f7c5c7 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
@@ -187,6 +187,7 @@ class StripedBlockReader {
 break;
   }
   n += nread;
+  stripedReader.getReconstructor().incrBytesRead(nread);
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
index a8e9d30..5554d68 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
 
 /**
  * StripedBlockReconstructor reconstruct one or more missed striped block in
@@ -66,7 +67,10 @@ class StripedBlockReconstructor extends StripedReconstructor
   getDatanode().getMetrics().incrECFailedReconstructionTasks();
 } finally {
   getDatanode().decrementXmitsInProgress();
-  getDatanode().getMetrics().incrECReconstructionTasks();
+  final DataNodeMetrics metrics = getDatanode().getMetrics();
+  metrics.incrECReconstructionTasks();
+  metrics.incrECReconstructionBytesRead(getBytesRead());
+  metrics.incrECReconstructionBytesWritten(getBytesWritten());
   getStripedReader().close();
   stripedWriter.close();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
index 592be45..d999202 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
@@ -196,6 +196,7 @@ class StripedBlockWriter {