This is an automated email from the ASF dual-hosted git repository.
mthakur pushed a change to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
from 4ba463069bd HADOOP-18288. Total requests and total requests per sec
served by RPC servers (#4485)
new 5c348c41ab8 HADOOP-11867. Add a high-performance vectored read API.
(#3904)
new 9f03f879633 HADOOP-18104: S3A: Add configs to configure
minSeekForVectorReads and maxReadSizeForVectorReads (#3964)
new bb5a17b1771 HADOOP-18107 Adding scale test for vectored reads for
large file (#4273)
new bfb7d020d10 HADOOP-18105 Implement buffer pooling with weak references
(#4263)
new c517b086f2c HADOOP-18106: Handle memory fragmentation in S3A Vectored
IO. (#4445)
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dev-support/Jenkinsfile | 2 +-
.../apache/hadoop/fs/BufferedFSInputStream.java | 27 +-
.../org/apache/hadoop/fs/ChecksumFileSystem.java | 213 +++++++++--
.../org/apache/hadoop/fs/FSDataInputStream.java | 22 +-
.../main/java/org/apache/hadoop/fs/FileRange.java | 67 ++++
.../org/apache/hadoop/fs/PositionedReadable.java | 41 ++-
.../org/apache/hadoop/fs/RawLocalFileSystem.java | 110 +++++-
.../org/apache/hadoop/fs/StreamCapabilities.java | 6 +
.../org/apache/hadoop/fs/VectoredReadUtils.java | 292 +++++++++++++++
.../apache/hadoop/fs/impl/CombinedFileRange.java | 70 ++++
.../org/apache/hadoop/fs/impl/FileRangeImpl.java | 74 ++++
.../java/org/apache/hadoop/io/ByteBufferPool.java | 5 +
.../apache/hadoop/io/ElasticByteBufferPool.java | 4 +-
.../io/WeakReferencedElasticByteBufferPool.java | 155 ++++++++
.../site/markdown/filesystem/fsdatainputstream.md | 39 ++
.../apache/hadoop/fs/TestVectoredReadUtils.java | 371 +++++++++++++++++++
.../contract/AbstractContractVectoredReadTest.java | 406 +++++++++++++++++++++
.../hadoop/fs/contract/ContractTestUtils.java | 84 +++++
.../localfs/TestLocalFSContractVectoredRead.java | 86 +++++
.../rawlocal/TestRawLocalContractVectoredRead.java | 35 ++
...estMoreWeakReferencedElasticByteBufferPool.java | 97 +++++
.../TestWeakReferencedElasticByteBufferPool.java | 232 ++++++++++++
.../java/org/apache/hadoop/test/MoreAsserts.java | 49 ++-
hadoop-common-project/pom.xml | 1 -
hadoop-project/pom.xml | 11 +
.../java/org/apache/hadoop/fs/s3a/Constants.java | 26 ++
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 39 +-
.../org/apache/hadoop/fs/s3a/S3AInputStream.java | 391 +++++++++++++++++++-
.../org/apache/hadoop/fs/s3a/S3AReadOpContext.java | 20 +-
.../apache/hadoop/fs/s3a/VectoredIOContext.java | 78 ++++
.../fs/s3a/impl/GetContentSummaryOperation.java | 3 +-
.../site/markdown/tools/hadoop-aws/performance.md | 30 ++
.../contract/s3a/ITestS3AContractVectoredRead.java | 159 ++++++++
.../hadoop/fs/s3a/TestS3AInputStreamRetry.java | 3 +-
.../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 32 ++
.../hadoop-aws/src/test/resources/log4j.properties | 2 +-
hadoop-tools/hadoop-benchmark/pom.xml | 94 +++++
.../hadoop-benchmark/src/main/assembly/uber.xml | 33 ++
.../hadoop-benchmark/src/main/findbugs/exclude.xml | 22 ++
.../hadoop/benchmark/VectoredReadBenchmark.java | 245 +++++++++++++
.../org/apache/hadoop/benchmark/package-info.java | 22 ++
hadoop-tools/pom.xml | 1 +
42 files changed, 3602 insertions(+), 97 deletions(-)
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileRange.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/CombinedFileRange.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileRangeImpl.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WeakReferencedElasticByteBufferPool.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractVectoredReadTest.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractVectoredRead.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawLocalContractVectoredRead.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMoreWeakReferencedElasticByteBufferPool.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWeakReferencedElasticByteBufferPool.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/VectoredIOContext.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractVectoredRead.java
create mode 100644 hadoop-tools/hadoop-benchmark/pom.xml
create mode 100644 hadoop-tools/hadoop-benchmark/src/main/assembly/uber.xml
create mode 100644 hadoop-tools/hadoop-benchmark/src/main/findbugs/exclude.xml
create mode 100644
hadoop-tools/hadoop-benchmark/src/main/java/org/apache/hadoop/benchmark/VectoredReadBenchmark.java
create mode 100644
hadoop-tools/hadoop-benchmark/src/main/java/org/apache/hadoop/benchmark/package-info.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
