This is an automated email from the ASF dual-hosted git repository.
stevel pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
from e4550e1aefc YARN-6476. Advanced Federation UI based on YARN UI v2.
(#5920) Contributed by Shilun Fan.
add 81d90fd65b3 HADOOP-18073. S3A: Upgrade AWS SDK to V2 (#5995)
No new revisions were added by this update.
Summary of changes:
LICENSE-binary | 1 +
.../hadoop/fs/statistics/StoreStatisticNames.java | 4 +
.../src/main/resources/core-default.xml | 52 +-
hadoop-project/pom.xml | 22 +-
.../hadoop-aws/dev-support/findbugs-exclude.xml | 5 +
hadoop-tools/hadoop-aws/pom.xml | 16 +-
.../hadoop/fs/s3a/AWSBadRequestException.java | 4 +-
.../apache/hadoop/fs/s3a/AWSClientIOException.java | 18 +-
.../hadoop/fs/s3a/AWSCredentialProviderList.java | 110 +-
.../hadoop/fs/s3a/AWSNoResponseException.java | 9 +-
.../apache/hadoop/fs/s3a/AWSRedirectException.java | 4 +-
.../org/apache/hadoop/fs/s3a/AWSS3IOException.java | 25 +-
.../hadoop/fs/s3a/AWSServiceIOException.java | 37 +-
.../fs/s3a/AWSServiceThrottledException.java | 9 +-
.../hadoop/fs/s3a/AWSStatus500Exception.java | 9 +-
.../fs/s3a/AnonymousAWSCredentialsProvider.java | 18 +-
.../java/org/apache/hadoop/fs/s3a/ArnResource.java | 12 +-
.../java/org/apache/hadoop/fs/s3a/Constants.java | 16 +-
.../fs/s3a/CredentialInitializationException.java | 18 +-
.../hadoop/fs/s3a/DefaultS3ClientFactory.java | 430 +++-----
.../hadoop/fs/s3a/FailureInjectionPolicy.java | 2 +-
.../hadoop/fs/s3a/InconsistentAmazonS3Client.java | 345 ------
.../hadoop/fs/s3a/InconsistentS3ClientFactory.java | 54 -
.../java/org/apache/hadoop/fs/s3a/Invoker.java | 21 +-
.../java/org/apache/hadoop/fs/s3a/Listing.java | 53 +-
.../org/apache/hadoop/fs/s3a/MultipartUtils.java | 49 +-
.../fs/s3a/ProgressableProgressListener.java | 46 +-
.../apache/hadoop/fs/s3a/S3ABlockOutputStream.java | 124 ++-
.../org/apache/hadoop/fs/s3a/S3ADataBlocks.java | 8 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 1099 +++++++++++---------
.../org/apache/hadoop/fs/s3a/S3AInputStream.java | 116 +--
.../org/apache/hadoop/fs/s3a/S3AInternals.java | 118 +++
.../org/apache/hadoop/fs/s3a/S3ARetryPolicy.java | 56 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 681 ++++--------
.../org/apache/hadoop/fs/s3a/S3ClientFactory.java | 187 +++-
.../org/apache/hadoop/fs/s3a/S3ListRequest.java | 16 +-
.../org/apache/hadoop/fs/s3a/S3ListResult.java | 61 +-
.../apache/hadoop/fs/s3a/S3ObjectAttributes.java | 27 -
.../fs/s3a/SharedInstanceCredentialProvider.java | 1 -
.../fs/s3a/SimpleAWSCredentialsProvider.java | 30 +-
.../java/org/apache/hadoop/fs/s3a/Statistic.java | 5 +
.../fs/s3a/TemporaryAWSCredentialsProvider.java | 7 +-
.../java/org/apache/hadoop/fs/s3a/Tristate.java | 45 +-
.../java/org/apache/hadoop/fs/s3a/UploadInfo.java | 12 +-
.../apache/hadoop/fs/s3a/WriteOperationHelper.java | 202 ++--
.../org/apache/hadoop/fs/s3a/WriteOperations.java | 100 +-
.../hadoop/fs/s3a/adapter/AwsV1BindingSupport.java | 118 +++
.../V1ToV2AwsCredentialProviderAdapter.java | 165 +++
.../{UploadInfo.java => adapter/package-info.java} | 32 +-
.../apache/hadoop/fs/s3a/api/RequestFactory.java | 224 ++--
.../fs/s3a/audit/AWSAuditEventCallbacks.java | 105 +-
.../hadoop/fs/s3a/audit/AWSRequestAnalyzer.java | 135 +--
.../hadoop/fs/s3a/audit/AuditIntegration.java | 54 +-
.../hadoop/fs/s3a/audit/AuditManagerS3A.java | 21 +-
.../hadoop/fs/s3a/audit/S3AAuditConstants.java | 13 +-
.../fs/s3a/audit/impl/ActiveAuditManagerS3A.java | 419 +++++---
.../hadoop/fs/s3a/audit/impl/LoggingAuditor.java | 125 ++-
.../fs/s3a/audit/impl/NoopAuditManagerS3A.java | 16 +-
.../s3a/audit/impl/S3AInternalAuditConstants.java | 16 +-
.../fs/s3a/auth/AbstractAWSCredentialProvider.java | 13 +-
.../auth/AbstractSessionCredentialsProvider.java | 28 +-
.../fs/s3a/auth/AssumedRoleCredentialProvider.java | 95 +-
.../fs/s3a/auth/CredentialProviderListFactory.java | 303 ++++++
.../s3a/auth/IAMInstanceCredentialsProvider.java | 49 +-
.../fs/s3a/auth/MarshalledCredentialBinding.java | 58 +-
.../fs/s3a/auth/MarshalledCredentialProvider.java | 5 +-
.../hadoop/fs/s3a/auth/NoAuthWithAWSException.java | 2 +-
.../hadoop/fs/s3a/auth/STSClientFactory.java | 141 ++-
.../apache/hadoop/fs/s3a/auth/SignerFactory.java | 114 ++
.../apache/hadoop/fs/s3a/auth/SignerManager.java | 8 +-
.../delegation/EncryptionSecretOperations.java | 38 +-
.../fs/s3a/auth/delegation/RoleTokenBinding.java | 2 +-
.../s3a/auth/delegation/S3ADelegationTokens.java | 5 -
.../s3a/auth/delegation/SessionTokenBinding.java | 61 +-
.../hadoop/fs/s3a/commit/AbstractS3ACommitter.java | 9 +-
.../apache/hadoop/fs/s3a/commit/PutTracker.java | 4 +-
.../fs/s3a/commit/files/SinglePendingCommit.java | 13 +-
.../fs/s3a/commit/impl/CommitOperations.java | 54 +-
.../fs/s3a/commit/magic/MagicCommitTracker.java | 28 +-
.../AWSCannedACL.java} | 28 +-
.../apache/hadoop/fs/s3a/impl/AWSClientConfig.java | 394 +++++++
.../org/apache/hadoop/fs/s3a/impl/AWSHeaders.java | 98 ++
.../hadoop/fs/s3a/impl/BulkDeleteRetryHandler.java | 12 +-
.../hadoop/fs/s3a/impl/ChangeDetectionPolicy.java | 125 ++-
.../apache/hadoop/fs/s3a/impl/ChangeTracker.java | 71 +-
.../s3a/impl/ConfigureShadedAWSSocketFactory.java | 13 +-
.../org/apache/hadoop/fs/s3a/impl/CopyOutcome.java | 80 --
.../apache/hadoop/fs/s3a/impl/DeleteOperation.java | 17 +-
.../hadoop/fs/s3a/impl/ErrorTranslation.java | 14 +-
.../hadoop/fs/s3a/impl/HeaderProcessing.java | 206 ++--
.../fs/s3a/impl/InstantiationIOException.java | 180 ++++
.../hadoop/fs/s3a/impl/InternalConstants.java | 51 +-
...upport.java => MultiObjectDeleteException.java} | 67 +-
.../apache/hadoop/fs/s3a/impl/NetworkBinding.java | 11 +-
.../hadoop/fs/s3a/impl/OperationCallbacks.java | 15 +-
.../{Tristate.java => impl/ProgressListener.java} | 14 +-
.../ProgressListenerEvent.java} | 17 +-
.../apache/hadoop/fs/s3a/impl/RenameOperation.java | 23 +-
.../hadoop/fs/s3a/impl/RequestFactoryImpl.java | 663 ++++++------
.../hadoop/fs/s3a/impl/S3AMultipartUploader.java | 31 +-
.../hadoop/fs/s3a/impl/SDKStreamDrainer.java | 43 +-
.../org/apache/hadoop/fs/s3a/impl/V2Migration.java | 72 +-
.../hadoop/fs/s3a/prefetch/S3ARemoteObject.java | 54 +-
.../fs/s3a/prefetch/S3ARemoteObjectReader.java | 7 +-
.../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 8 +-
.../hadoop/fs/s3a/select/BlockingEnumeration.java | 156 +++
.../apache/hadoop/fs/s3a/select/SelectBinding.java | 126 ++-
.../fs/s3a/select/SelectEventStreamPublisher.java | 124 +++
.../hadoop/fs/s3a/select/SelectInputStream.java | 32 +-
.../fs/s3a/select/SelectObjectContentHelper.java | 114 ++
.../statistics/impl/AwsStatisticsCollector.java | 166 ++-
.../org/apache/hadoop/fs/s3a/tools/MarkerTool.java | 17 +-
.../hadoop/fs/s3a/tools/MarkerToolOperations.java | 13 +-
.../fs/s3a/tools/MarkerToolOperationsImpl.java | 11 +-
.../markdown/tools/hadoop-aws/assumed_roles.md | 2 +-
.../src/site/markdown/tools/hadoop-aws/auditing.md | 48 +-
.../markdown/tools/hadoop-aws/aws_sdk_upgrade.md | 340 +++++-
.../tools/hadoop-aws/aws_sdk_v2_changelog.md | 340 ++++++
.../markdown/tools/hadoop-aws/delegation_tokens.md | 2 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 142 +--
.../site/markdown/tools/hadoop-aws/s3_select.md | 43 +-
.../src/site/markdown/tools/hadoop-aws/testing.md | 69 +-
.../tools/hadoop-aws/troubleshooting_s3a.md | 95 +-
.../apache/hadoop/fs/s3a/AbstractS3AMockTest.java | 30 +-
.../apache/hadoop/fs/s3a/AbstractS3ATestBase.java | 8 +
.../apache/hadoop/fs/s3a/EncryptionTestUtils.java | 20 +-
.../fs/s3a/ITestS3AAWSCredentialsProvider.java | 164 +--
.../hadoop/fs/s3a/ITestS3ABucketExistence.java | 41 +-
.../apache/hadoop/fs/s3a/ITestS3ACannedACLs.java | 31 +-
.../fs/s3a/ITestS3AClientSideEncryptionKms.java | 6 +-
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 236 +++--
.../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 2 +-
.../fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java | 8 +-
.../ITestS3AEncryptionWithDefaultS3Settings.java | 3 +-
.../hadoop/fs/s3a/ITestS3AEndpointRegion.java | 200 ++--
.../hadoop/fs/s3a/ITestS3AFailureHandling.java | 28 +-
.../hadoop/fs/s3a/ITestS3AMiscOperations.java | 35 +-
.../hadoop/fs/s3a/ITestS3AMultipartUtils.java | 5 +-
.../hadoop/fs/s3a/ITestS3ARequesterPays.java | 2 +-
.../apache/hadoop/fs/s3a/ITestS3AStorageClass.java | 1 -
.../fs/s3a/ITestS3ATemporaryCredentials.java | 30 +-
.../apache/hadoop/fs/s3a/MockS3AFileSystem.java | 17 +-
.../apache/hadoop/fs/s3a/MockS3ClientFactory.java | 49 +-
.../apache/hadoop/fs/s3a/MultipartTestUtils.java | 35 +-
.../org/apache/hadoop/fs/s3a/S3ATestConstants.java | 5 +
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 17 +-
.../org/apache/hadoop/fs/s3a/TestArnResource.java | 9 +-
.../java/org/apache/hadoop/fs/s3a/TestInvoker.java | 105 +-
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 266 ++---
.../hadoop/fs/s3a/TestS3ABlockOutputStream.java | 11 +-
.../apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java | 25 +-
.../hadoop/fs/s3a/TestS3AExceptionTranslation.java | 208 +++-
.../apache/hadoop/fs/s3a/TestS3AGetFileStatus.java | 95 +-
.../hadoop/fs/s3a/TestS3AInputStreamRetry.java | 130 ++-
.../org/apache/hadoop/fs/s3a/TestS3AProxy.java | 18 +-
.../org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java | 50 +-
.../hadoop/fs/s3a/TestStreamChangeTracker.java | 107 +-
.../fs/s3a/TestWildflyAndOpenSSLBinding.java | 9 +-
.../fs/s3a/adapter/TestV1CredentialsProvider.java | 222 ++++
.../hadoop/fs/s3a/audit/AbstractAuditingTest.java | 98 +-
.../hadoop/fs/s3a/audit/AuditTestSupport.java | 4 +-
.../hadoop/fs/s3a/audit/ITestAuditManager.java | 18 +-
...ler.java => SimpleAWSExecutionInterceptor.java} | 37 +-
.../hadoop/fs/s3a/audit/TestAuditIntegration.java | 82 +-
.../fs/s3a/audit/TestAuditSpanLifecycle.java | 11 +-
.../fs/s3a/audit/TestHttpReferrerAuditHeader.java | 55 +-
.../hadoop/fs/s3a/audit/TestLoggingAuditor.java | 41 +-
.../apache/hadoop/fs/s3a/auth/ITestAssumeRole.java | 36 +-
.../hadoop/fs/s3a/auth/ITestCustomSigner.java | 47 +-
.../apache/hadoop/fs/s3a/auth/RoleTestUtils.java | 7 +-
.../fs/s3a/auth/TestMarshalledCredentials.java | 10 +-
.../hadoop/fs/s3a/auth/TestSignerManager.java | 41 +-
.../auth/delegation/CountInvocationsProvider.java | 34 +-
.../ITestSessionDelegationInFilesystem.java | 30 +-
.../delegation/ITestSessionDelegationTokens.java | 28 +-
.../delegation/TestS3ADelegationTokenSupport.java | 1 -
.../fs/s3a/commit/AbstractITCommitProtocol.java | 4 +-
.../fs/s3a/commit/staging/StagingTestBase.java | 178 ++--
.../staging/TestDirectoryCommitterScale.java | 7 +-
.../s3a/commit/staging/TestStagingCommitter.java | 31 +-
.../staging/TestStagingPartitionedTaskCommit.java | 9 +-
.../fileContext/ITestS3AFileContextStatistics.java | 3 +-
.../fs/s3a/impl/ITestPartialRenamesDeletes.java | 1 -
.../hadoop/fs/s3a/impl/ITestRenameDeleteRace.java | 5 +-
.../hadoop/fs/s3a/impl/TestHeaderProcessing.java | 32 +-
.../hadoop/fs/s3a/impl/TestNetworkBinding.java | 43 -
.../hadoop/fs/s3a/impl/TestRequestFactory.java | 109 +-
.../hadoop/fs/s3a/impl/TestSDKStreamDrainer.java | 16 +-
.../performance/ITestDirectoryMarkerListing.java | 25 +-
.../fs/s3a/prefetch/MockS3ARemoteObject.java | 27 +-
.../hadoop/fs/s3a/prefetch/S3APrefetchFakes.java | 48 +-
.../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 25 +-
.../scale/ILoadTestS3ABulkDeleteThrottling.java | 13 +-
.../fs/s3a/scale/ITestS3ADirectoryPerformance.java | 22 +-
.../s3a/scale/ITestS3AHugeFilesStorageClass.java | 4 +-
.../hadoop/fs/s3a/select/AbstractS3SelectTest.java | 4 +-
.../hadoop/fs/s3a/select/ITestS3SelectLandsat.java | 2 +-
.../hadoop/fs/s3a/select/StreamPublisher.java | 89 ++
.../fs/s3a/select/TestBlockingEnumeration.java | 200 ++++
.../s3a/select/TestSelectEventStreamPublisher.java | 190 ++++
.../statistics/ITestAWSStatisticCollection.java | 4 +-
.../apache/hadoop/fs/s3a/test/ExtraAssertions.java | 2 +-
.../fs/s3a/test/MinimalOperationCallbacks.java | 14 +-
.../test/MinimalWriteOperationHelperCallbacks.java | 16 +-
.../hadoop/fs/s3a/tools/ITestMarkerTool.java | 7 -
205 files changed, 8899 insertions(+), 5950 deletions(-)
delete mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
delete mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/AwsV1BindingSupport.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/V1ToV2AwsCredentialProviderAdapter.java
copy
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/{UploadInfo.java
=> adapter/package-info.java} (59%)
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
copy
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/{AWSNoResponseException.java
=> impl/AWSCannedACL.java} (62%)
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSHeaders.java
delete mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CopyOutcome.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InstantiationIOException.java
rename
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/{MultiObjectDeleteSupport.java
=> MultiObjectDeleteException.java} (62%)
copy
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/{Tristate.java
=> impl/ProgressListener.java} (72%)
copy
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/{Tristate.java
=> impl/ProgressListenerEvent.java} (72%)
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/BlockingEnumeration.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectEventStreamPublisher.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectObjectContentHelper.java
create mode 100644
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_v2_changelog.md
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/adapter/TestV1CredentialsProvider.java
rename
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/{SimpleAWSRequestHandler.java
=> SimpleAWSExecutionInterceptor.java} (56%)
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/StreamPublisher.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/TestBlockingEnumeration.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/TestSelectEventStreamPublisher.java
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
