This is an automated email from the ASF dual-hosted git repository.
stevel pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
from 17d64ba495e Revert "HDFS-16488. [SPS]: Expose metrics to JMX for
external SPS (#4035)" (#4232)
new 1b4dba99b5d HADOOP-16202. Enhanced openFile(): hadoop-common changes.
(#2584/1)
new 6999acf5200 HADOOP-16202. Enhanced openFile(): mapreduce and YARN
changes. (#2584/2)
new e0cd0a82e03 HADOOP-16202. Enhanced openFile(): hadoop-aws changes.
(#2584/3)
new 44ae2fa8e53 HADOOP-16202. Enhanced openFile(): hadoop-azure changes.
(#2584/4)
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/hadoop/fs/AvroFSInput.java | 11 +-
.../org/apache/hadoop/fs/ChecksumFileSystem.java | 4 +-
.../main/java/org/apache/hadoop/fs/FSBuilder.java | 14 +
.../java/org/apache/hadoop/fs/FileContext.java | 18 +-
.../main/java/org/apache/hadoop/fs/FileSystem.java | 11 +-
.../main/java/org/apache/hadoop/fs/FileUtil.java | 61 ++-
.../hadoop/fs/FutureDataInputStreamBuilder.java | 8 +-
.../main/java/org/apache/hadoop/fs/Options.java | 119 ++++
.../hadoop/fs/impl/AbstractFSBuilderImpl.java | 38 +-
.../fs/impl/FileSystemMultipartUploader.java | 9 +-
.../fs/impl/FutureDataInputStreamBuilderImpl.java | 8 +-
.../org/apache/hadoop/fs/impl/FutureIOSupport.java | 83 +--
.../apache/hadoop/fs/impl/OpenFileParameters.java | 13 +
.../apache/hadoop/fs/impl/WrappedIOException.java | 11 +-
.../hadoop/fs/shell/CommandWithDestination.java | 9 +-
.../org/apache/hadoop/fs/shell/CopyCommands.java | 3 +-
.../java/org/apache/hadoop/fs/shell/Display.java | 3 +-
.../main/java/org/apache/hadoop/fs/shell/Head.java | 8 +-
.../java/org/apache/hadoop/fs/shell/PathData.java | 35 ++
.../main/java/org/apache/hadoop/fs/shell/Tail.java | 11 +-
.../hadoop/fs/statistics/StoreStatisticNames.java | 9 +
.../hadoop/fs/statistics/StreamStatisticNames.java | 19 +-
.../fs/statistics/impl/IOStatisticsBinding.java | 44 +-
.../java/org/apache/hadoop/io/SequenceFile.java | 14 +-
.../org/apache/hadoop/util/JsonSerialization.java | 6 +-
.../util/functional/CommonCallableSupplier.java | 2 +-
.../apache/hadoop/util/functional/FutureIO.java | 90 ++++
.../src/site/markdown/filesystem/filesystem.md | 90 +---
.../filesystem/fsdatainputstreambuilder.md | 588 +++++++++++++++++++-
.../src/site/markdown/filesystem/index.md | 1 +
.../src/site/markdown/filesystem/openfile.md | 122 +++++
.../AbstractContractMultipartUploaderTest.java | 2 +-
.../fs/contract/AbstractContractOpenTest.java | 80 ++-
.../hadoop/fs/contract/ContractTestUtils.java | 13 +-
.../fs/statistics/IOStatisticAssertions.java | 20 +
.../hadoop/fs/statistics/TestDurationTracking.java | 3 +-
.../jobhistory/JobHistoryCopyService.java | 10 +-
.../org/apache/hadoop/mapred/LineRecordReader.java | 13 +-
.../lib/input/FixedLengthRecordReader.java | 7 +-
.../mapreduce/lib/input/LineRecordReader.java | 14 +-
.../mapreduce/lib/input/NLineInputFormat.java | 6 +-
.../hadoop/examples/terasort/TeraInputFormat.java | 16 +-
.../hadoop-aws/dev-support/findbugs-exclude.xml | 5 +
.../java/org/apache/hadoop/fs/s3a/Constants.java | 45 +-
.../java/org/apache/hadoop/fs/s3a/Invoker.java | 28 +
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 283 +++++-----
.../org/apache/hadoop/fs/s3a/S3AInputPolicy.java | 93 +++-
.../org/apache/hadoop/fs/s3a/S3AInputStream.java | 263 ++++++---
.../apache/hadoop/fs/s3a/S3AInstrumentation.java | 12 +-
.../org/apache/hadoop/fs/s3a/S3AReadOpContext.java | 101 +++-
.../apache/hadoop/fs/s3a/S3ObjectAttributes.java | 15 +-
.../java/org/apache/hadoop/fs/s3a/Statistic.java | 17 +
.../hadoop/fs/s3a/commit/CommitOperations.java | 8 +-
.../hadoop/fs/s3a/commit/files/PendingSet.java | 26 +-
.../fs/s3a/commit/files/SinglePendingCommit.java | 20 +-
.../hadoop/fs/s3a/impl/AbstractStoreOperation.java | 26 +-
.../hadoop/fs/s3a/impl/CallableSupplier.java | 2 +-
.../hadoop/fs/s3a/impl/InternalConstants.java | 21 +-
.../apache/hadoop/fs/s3a/impl/OpenFileSupport.java | 600 +++++++++++++++++++++
.../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 4 +-
.../fs/s3a/select/InternalSelectConstants.java | 2 +-
.../apache/hadoop/fs/s3a/select/SelectTool.java | 4 +-
.../s3a/statistics/S3AInputStreamStatistics.java | 7 +
.../statistics/impl/EmptyS3AStatisticsContext.java | 4 +
.../src/site/markdown/tools/hadoop-aws/index.md | 8 +
.../fs/contract/s3a/ITestS3AContractOpen.java | 67 +++
.../fs/contract/s3a/ITestS3AContractSeek.java | 15 +-
.../apache/hadoop/fs/s3a/AbstractS3AMockTest.java | 4 +
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 11 -
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 4 +-
.../hadoop/fs/s3a/TestS3AInputStreamRetry.java | 17 +-
.../org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java | 5 +-
.../hadoop/fs/s3a/TestStreamChangeTracker.java | 2 +-
.../hadoop/fs/s3a/impl/TestOpenFileSupport.java | 429 +++++++++++++++
.../fs/s3a/performance/ITestS3AOpenCost.java | 209 +++++++
.../hadoop/fs/s3a/performance/OperationCost.java | 6 +
.../s3a/scale/ITestS3AInputStreamPerformance.java | 58 +-
.../hadoop/fs/s3a/select/AbstractS3SelectTest.java | 2 +-
.../apache/hadoop/fs/s3a/select/ITestS3Select.java | 7 +-
.../hadoop/fs/s3a/select/ITestS3SelectMRJob.java | 4 +-
.../fs/azurebfs/AzureBlobFileSystemStore.java | 2 +-
.../contract/ITestAbfsFileSystemContractSeek.java | 2 +-
.../tools/mapred/RetriableFileCopyCommand.java | 9 +-
.../streaming/mapreduce/StreamInputFormat.java | 6 +-
.../yarn/logaggregation/AggregatedLogFormat.java | 17 +-
.../org/apache/hadoop/yarn/util/FSDownload.java | 24 +-
86 files changed, 3478 insertions(+), 662 deletions(-)
create mode 100644
hadoop-common-project/hadoop-common/src/site/markdown/filesystem/openfile.md
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OpenFileSupport.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestOpenFileSupport.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3AOpenCost.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
