This is an automated email from the ASF dual-hosted git repository.
stevel pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
from e6ecc4f3e44 YARN-11188. Only files belong to the first file controller
are removed even if multiple log aggregation file controllers are configured.
Contributed by Szilard Nemeth.
new 2daf0a814f8 HADOOP-11867. Add a high-performance vectored read API.
(#3904)
new 5db0f34e29c HADOOP-18104: S3A: Add configs to configure
minSeekForVectorReads and maxReadSizeForVectorReads (#3964)
new 1408dd89a74 HADOOP-18107 Adding scale test for vectored reads for
large file (#4273)
new 0d49bd20044 HADOOP-18105 Implement buffer pooling with weak references
(#4263)
new 4d1f6f9b995 HADOOP-18106: Handle memory fragmentation in S3A Vectored
IO. (#4445)
new e1842b2a749 HADOOP-18103. Add a high-performance vectored read API.
(#4476)
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dev-support/Jenkinsfile | 2 +-
.../apache/hadoop/fs/BufferedFSInputStream.java | 27 +-
.../org/apache/hadoop/fs/ChecksumFileSystem.java | 213 +++++++++--
.../org/apache/hadoop/fs/FSDataInputStream.java | 22 +-
.../main/java/org/apache/hadoop/fs/FileRange.java | 67 ++++
.../org/apache/hadoop/fs/PositionedReadable.java | 41 ++-
.../org/apache/hadoop/fs/RawLocalFileSystem.java | 110 +++++-
.../org/apache/hadoop/fs/StreamCapabilities.java | 6 +
.../org/apache/hadoop/fs/VectoredReadUtils.java | 292 +++++++++++++++
.../apache/hadoop/fs/impl/CombinedFileRange.java | 70 ++++
.../org/apache/hadoop/fs/impl/FileRangeImpl.java | 74 ++++
.../java/org/apache/hadoop/io/ByteBufferPool.java | 5 +
.../apache/hadoop/io/ElasticByteBufferPool.java | 4 +-
.../io/WeakReferencedElasticByteBufferPool.java | 155 ++++++++
.../site/markdown/filesystem/fsdatainputstream.md | 39 ++
.../apache/hadoop/fs/TestVectoredReadUtils.java | 371 +++++++++++++++++++
.../contract/AbstractContractVectoredReadTest.java | 406 +++++++++++++++++++++
.../hadoop/fs/contract/ContractTestUtils.java | 84 +++++
.../localfs/TestLocalFSContractVectoredRead.java | 86 +++++
.../rawlocal/TestRawLocalContractVectoredRead.java | 35 ++
...estMoreWeakReferencedElasticByteBufferPool.java | 97 +++++
.../TestWeakReferencedElasticByteBufferPool.java | 232 ++++++++++++
.../java/org/apache/hadoop/test/MoreAsserts.java | 49 ++-
hadoop-common-project/pom.xml | 1 -
hadoop-project/pom.xml | 11 +
.../java/org/apache/hadoop/fs/s3a/Constants.java | 26 ++
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 39 +-
.../org/apache/hadoop/fs/s3a/S3AInputStream.java | 391 +++++++++++++++++++-
.../org/apache/hadoop/fs/s3a/S3AReadOpContext.java | 20 +-
.../apache/hadoop/fs/s3a/VectoredIOContext.java | 78 ++++
.../fs/s3a/impl/GetContentSummaryOperation.java | 3 +-
.../site/markdown/tools/hadoop-aws/performance.md | 30 ++
.../contract/s3a/ITestS3AContractVectoredRead.java | 159 ++++++++
.../hadoop/fs/s3a/TestS3AInputStreamRetry.java | 3 +-
.../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 32 ++
.../hadoop-aws/src/test/resources/log4j.properties | 2 +-
hadoop-tools/hadoop-benchmark/pom.xml | 94 +++++
.../hadoop-benchmark/src/main/assembly/uber.xml | 33 ++
.../hadoop-benchmark/src/main/findbugs/exclude.xml | 22 ++
.../hadoop/benchmark/VectoredReadBenchmark.java | 245 +++++++++++++
.../org/apache/hadoop/benchmark/package-info.java | 22 ++
hadoop-tools/pom.xml | 1 +
pom.xml | 1 +
43 files changed, 3603 insertions(+), 97 deletions(-)
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileRange.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/CombinedFileRange.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileRangeImpl.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WeakReferencedElasticByteBufferPool.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractVectoredReadTest.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractVectoredRead.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawLocalContractVectoredRead.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMoreWeakReferencedElasticByteBufferPool.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWeakReferencedElasticByteBufferPool.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/VectoredIOContext.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractVectoredRead.java
create mode 100644 hadoop-tools/hadoop-benchmark/pom.xml
create mode 100644 hadoop-tools/hadoop-benchmark/src/main/assembly/uber.xml
create mode 100644 hadoop-tools/hadoop-benchmark/src/main/findbugs/exclude.xml
create mode 100644
hadoop-tools/hadoop-benchmark/src/main/java/org/apache/hadoop/benchmark/VectoredReadBenchmark.java
create mode 100644
hadoop-tools/hadoop-benchmark/src/main/java/org/apache/hadoop/benchmark/package-info.java
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
