[10/29] hadoop git commit: HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by Sammi Chen and Andrew Wang
HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by Sammi Chen and Andrew Wang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f14f6d0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f14f6d0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f14f6d0 Branch: refs/heads/YARN-5085 Commit: 1f14f6d038aecad55a5398c6fa4137c9d2f44729 Parents: ada876c Author: Kai ZhengAuthored: Wed Dec 14 14:50:50 2016 +0800 Committer: Kai Zheng Committed: Wed Dec 14 14:50:50 2016 +0800 -- .../erasurecode/StripedBlockReader.java | 1 + .../erasurecode/StripedBlockReconstructor.java | 6 +- .../erasurecode/StripedBlockWriter.java | 1 + .../datanode/erasurecode/StripedReader.java | 4 + .../erasurecode/StripedReconstructor.java | 21 +++ .../datanode/erasurecode/StripedWriter.java | 4 + .../datanode/metrics/DataNodeMetrics.java | 18 ++- .../apache/hadoop/hdfs/StripedFileTestUtil.java | 24 +++ .../TestDataNodeErasureCodingMetrics.java | 149 +-- 9 files changed, 147 insertions(+), 81 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java index a27de9b..0f7c5c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java @@ -187,6 +187,7 @@ class StripedBlockReader { break; } n += nread; + stripedReader.getReconstructor().incrBytesRead(nread); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java index a8e9d30..5554d68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics; /** * StripedBlockReconstructor reconstruct one or more missed striped block in @@ -66,7 +67,10 @@ class StripedBlockReconstructor extends StripedReconstructor getDatanode().getMetrics().incrECFailedReconstructionTasks(); } finally { getDatanode().decrementXmitsInProgress(); - getDatanode().getMetrics().incrECReconstructionTasks(); + final DataNodeMetrics metrics = getDatanode().getMetrics(); + metrics.incrECReconstructionTasks(); + metrics.incrECReconstructionBytesRead(getBytesRead()); + metrics.incrECReconstructionBytesWritten(getBytesWritten()); getStripedReader().close(); stripedWriter.close(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java index 592be45..d999202 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java @@ -196,6 +196,7 @@ class StripedBlockWriter { packet.writeTo(targetOutputStream); blockOffset4Target += toWrite; +
[34/50] [abbrv] hadoop git commit: HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by Sammi Chen and Andrew Wang
HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by Sammi Chen and Andrew Wang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f14f6d0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f14f6d0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f14f6d0 Branch: refs/heads/HDFS-10285 Commit: 1f14f6d038aecad55a5398c6fa4137c9d2f44729 Parents: ada876c Author: Kai ZhengAuthored: Wed Dec 14 14:50:50 2016 +0800 Committer: Kai Zheng Committed: Wed Dec 14 14:50:50 2016 +0800 -- .../erasurecode/StripedBlockReader.java | 1 + .../erasurecode/StripedBlockReconstructor.java | 6 +- .../erasurecode/StripedBlockWriter.java | 1 + .../datanode/erasurecode/StripedReader.java | 4 + .../erasurecode/StripedReconstructor.java | 21 +++ .../datanode/erasurecode/StripedWriter.java | 4 + .../datanode/metrics/DataNodeMetrics.java | 18 ++- .../apache/hadoop/hdfs/StripedFileTestUtil.java | 24 +++ .../TestDataNodeErasureCodingMetrics.java | 149 +-- 9 files changed, 147 insertions(+), 81 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java index a27de9b..0f7c5c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java @@ -187,6 +187,7 @@ class StripedBlockReader { break; } n += nread; + stripedReader.getReconstructor().incrBytesRead(nread); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java index a8e9d30..5554d68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics; /** * StripedBlockReconstructor reconstruct one or more missed striped block in @@ -66,7 +67,10 @@ class StripedBlockReconstructor extends StripedReconstructor getDatanode().getMetrics().incrECFailedReconstructionTasks(); } finally { getDatanode().decrementXmitsInProgress(); - getDatanode().getMetrics().incrECReconstructionTasks(); + final DataNodeMetrics metrics = getDatanode().getMetrics(); + metrics.incrECReconstructionTasks(); + metrics.incrECReconstructionBytesRead(getBytesRead()); + metrics.incrECReconstructionBytesWritten(getBytesWritten()); getStripedReader().close(); stripedWriter.close(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java index 592be45..d999202 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java @@ -196,6 +196,7 @@ class StripedBlockWriter { packet.writeTo(targetOutputStream); blockOffset4Target += toWrite; +
[5/9] hadoop git commit: HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by Sammi Chen and Andrew Wang
HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by Sammi Chen and Andrew Wang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f14f6d0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f14f6d0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f14f6d0 Branch: refs/heads/YARN-5734 Commit: 1f14f6d038aecad55a5398c6fa4137c9d2f44729 Parents: ada876c Author: Kai ZhengAuthored: Wed Dec 14 14:50:50 2016 +0800 Committer: Kai Zheng Committed: Wed Dec 14 14:50:50 2016 +0800 -- .../erasurecode/StripedBlockReader.java | 1 + .../erasurecode/StripedBlockReconstructor.java | 6 +- .../erasurecode/StripedBlockWriter.java | 1 + .../datanode/erasurecode/StripedReader.java | 4 + .../erasurecode/StripedReconstructor.java | 21 +++ .../datanode/erasurecode/StripedWriter.java | 4 + .../datanode/metrics/DataNodeMetrics.java | 18 ++- .../apache/hadoop/hdfs/StripedFileTestUtil.java | 24 +++ .../TestDataNodeErasureCodingMetrics.java | 149 +-- 9 files changed, 147 insertions(+), 81 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java index a27de9b..0f7c5c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java @@ -187,6 +187,7 @@ class StripedBlockReader { break; } n += nread; + stripedReader.getReconstructor().incrBytesRead(nread); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java index a8e9d30..5554d68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics; /** * StripedBlockReconstructor reconstruct one or more missed striped block in @@ -66,7 +67,10 @@ class StripedBlockReconstructor extends StripedReconstructor getDatanode().getMetrics().incrECFailedReconstructionTasks(); } finally { getDatanode().decrementXmitsInProgress(); - getDatanode().getMetrics().incrECReconstructionTasks(); + final DataNodeMetrics metrics = getDatanode().getMetrics(); + metrics.incrECReconstructionTasks(); + metrics.incrECReconstructionBytesRead(getBytesRead()); + metrics.incrECReconstructionBytesWritten(getBytesWritten()); getStripedReader().close(); stripedWriter.close(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java index 592be45..d999202 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java @@ -196,6 +196,7 @@ class StripedBlockWriter { packet.writeTo(targetOutputStream); blockOffset4Target += toWrite; +
hadoop git commit: HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by Sammi Chen and Andrew Wang
Repository: hadoop Updated Branches: refs/heads/trunk ada876cd1 -> 1f14f6d03 HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by Sammi Chen and Andrew Wang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f14f6d0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f14f6d0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f14f6d0 Branch: refs/heads/trunk Commit: 1f14f6d038aecad55a5398c6fa4137c9d2f44729 Parents: ada876c Author: Kai ZhengAuthored: Wed Dec 14 14:50:50 2016 +0800 Committer: Kai Zheng Committed: Wed Dec 14 14:50:50 2016 +0800 -- .../erasurecode/StripedBlockReader.java | 1 + .../erasurecode/StripedBlockReconstructor.java | 6 +- .../erasurecode/StripedBlockWriter.java | 1 + .../datanode/erasurecode/StripedReader.java | 4 + .../erasurecode/StripedReconstructor.java | 21 +++ .../datanode/erasurecode/StripedWriter.java | 4 + .../datanode/metrics/DataNodeMetrics.java | 18 ++- .../apache/hadoop/hdfs/StripedFileTestUtil.java | 24 +++ .../TestDataNodeErasureCodingMetrics.java | 149 +-- 9 files changed, 147 insertions(+), 81 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java index a27de9b..0f7c5c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java @@ -187,6 +187,7 @@ class StripedBlockReader { break; } n += nread; + stripedReader.getReconstructor().incrBytesRead(nread); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java index a8e9d30..5554d68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics; /** * StripedBlockReconstructor reconstruct one or more missed striped block in @@ -66,7 +67,10 @@ class StripedBlockReconstructor extends StripedReconstructor getDatanode().getMetrics().incrECFailedReconstructionTasks(); } finally { getDatanode().decrementXmitsInProgress(); - getDatanode().getMetrics().incrECReconstructionTasks(); + final DataNodeMetrics metrics = getDatanode().getMetrics(); + metrics.incrECReconstructionTasks(); + metrics.incrECReconstructionBytesRead(getBytesRead()); + metrics.incrECReconstructionBytesWritten(getBytesWritten()); getStripedReader().close(); stripedWriter.close(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java index 592be45..d999202 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java @@ -196,6 +196,7 @@ class StripedBlockWriter {