HADOOP-13735 ITestS3AFileContextStatistics.testStatistics() failing. Contributed by Pieter Reuse
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ae270af Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ae270af Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ae270af Branch: refs/heads/HDFS-7240 Commit: 9ae270af02c243993f853513c731cb268430e492 Parents: 73504b1 Author: Steve Loughran <ste...@apache.org> Authored: Thu Oct 20 14:50:30 2016 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Thu Oct 20 14:50:59 2016 +0100 ---------------------------------------------------------------------- .../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 +- .../org/apache/hadoop/fs/s3a/S3AInstrumentation.java | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ae270af/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 2354819..9908ba7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -548,7 +548,7 @@ public class S3AFileSystem extends FileSystem { progress, partSize, blockFactory, - instrumentation.newOutputStreamStatistics(), + instrumentation.newOutputStreamStatistics(statistics), new WriteOperationHelper(key) ), null); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ae270af/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java index 963c53f..fb8c852 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java @@ -37,6 +37,7 @@ import java.util.HashMap; import java.util.Map; import java.util.UUID; import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.fs.FileSystem.Statistics; import static org.apache.hadoop.fs.s3a.Statistic.*; @@ -639,9 +640,8 @@ public class S3AInstrumentation { * Create a stream output statistics instance. * @return the new instance */ - - OutputStreamStatistics newOutputStreamStatistics() { - return new OutputStreamStatistics(); + OutputStreamStatistics newOutputStreamStatistics(Statistics statistics) { + return new OutputStreamStatistics(statistics); } /** @@ -677,6 +677,12 @@ public class S3AInstrumentation { private final AtomicLong queueDuration = new AtomicLong(0); private final AtomicLong exceptionsInMultipartFinalize = new AtomicLong(0); + private Statistics statistics; + + public OutputStreamStatistics(Statistics statistics){ + this.statistics = statistics; + } + /** * Block is queued for upload. */ @@ -717,6 +723,7 @@ public class S3AInstrumentation { /** Intermediate report of bytes uploaded. */ void bytesTransferred(long byteCount) { bytesUploaded.addAndGet(byteCount); + statistics.incrementBytesWritten(byteCount); bytesPendingUpload.addAndGet(-byteCount); incrementGauge(STREAM_WRITE_BLOCK_UPLOADS_DATA_PENDING, -byteCount); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org