Merge branch 'HADOOP-15407' into trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0def6148 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0def6148 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0def6148 Branch: refs/heads/trunk Commit: 0def61482b7636161b0edcfee377c74f398953d3 Parents: 4758b4b d0b4624 Author: Sean Mackrory <[email protected]> Authored: Sat Sep 22 21:19:12 2018 -0600 Committer: Sean Mackrory <[email protected]> Committed: Sat Sep 22 21:19:12 2018 -0600 ---------------------------------------------------------------------- .gitignore | 1 + .../fs/CommonConfigurationKeysPublic.java | 4 +- .../src/main/resources/core-default.xml | 41 +- .../src/site/markdown/filesystem/filesystem.md | 11 +- .../conf/TestCommonConfigurationFields.java | 3 + .../apache/hadoop/conf/TestConfigRedactor.java | 7 + .../fs/contract/AbstractContractConcatTest.java | 34 +- .../AbstractContractGetFileStatusTest.java | 26 +- .../fs/contract/AbstractContractMkdirTest.java | 8 + .../hadoop/fs/contract/AbstractFSContract.java | 2 - .../hadoop/fs/contract/ContractTestUtils.java | 19 +- hadoop-project/pom.xml | 18 + hadoop-tools/hadoop-azure/pom.xml | 403 +++++- .../src/config/checkstyle-suppressions.xml | 49 + .../fs/azure/ClientThrottlingAnalyzer.java | 2 +- .../org/apache/hadoop/fs/azurebfs/Abfs.java | 46 + .../hadoop/fs/azurebfs/AbfsConfiguration.java | 576 ++++++++ .../org/apache/hadoop/fs/azurebfs/Abfss.java | 46 + .../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 953 +++++++++++++ .../fs/azurebfs/AzureBlobFileSystemStore.java | 1028 ++++++++++++++ .../fs/azurebfs/SecureAzureBlobFileSystem.java | 39 + .../azurebfs/constants/AbfsHttpConstants.java | 91 ++ .../azurebfs/constants/ConfigurationKeys.java | 89 ++ .../constants/FileSystemConfigurations.java | 67 + .../constants/FileSystemUriSchemes.java | 42 + .../constants/HttpHeaderConfigurations.java | 63 + .../fs/azurebfs/constants/HttpQueryParams.java | 40 + .../fs/azurebfs/constants/package-info.java | 22 + .../ConfigurationValidationAnnotations.java | 104 ++ .../contracts/annotations/package-info.java | 22 + .../diagnostics/ConfigurationValidator.java | 37 + .../contracts/diagnostics/package-info.java | 22 + .../exceptions/AbfsRestOperationException.java | 84 ++ .../AzureBlobFileSystemException.java | 56 + .../ConfigurationPropertyNotFoundException.java | 32 + .../FileSystemOperationUnhandledException.java | 33 + .../InvalidAbfsRestOperationException.java | 40 + .../InvalidAclOperationException.java | 33 + .../InvalidConfigurationValueException.java | 37 + .../InvalidFileSystemPropertyException.java | 33 + .../InvalidUriAuthorityException.java | 33 + .../exceptions/InvalidUriException.java | 33 + .../exceptions/KeyProviderException.java | 42 + .../contracts/exceptions/TimeoutException.java | 33 + .../TokenAccessProviderException.java | 36 + .../contracts/exceptions/package-info.java | 22 + .../fs/azurebfs/contracts/package-info.java | 22 + .../services/AzureServiceErrorCode.java | 115 ++ .../services/ListResultEntrySchema.java | 239 ++++ .../contracts/services/ListResultSchema.java | 58 + .../contracts/services/ReadBufferStatus.java | 29 + .../contracts/services/package-info.java | 22 + ...Base64StringConfigurationBasicValidator.java | 50 + .../BooleanConfigurationBasicValidator.java | 50 + .../ConfigurationBasicValidator.java | 67 + .../IntegerConfigurationBasicValidator.java | 68 + .../LongConfigurationBasicValidator.java | 63 + .../StringConfigurationBasicValidator.java | 43 + .../fs/azurebfs/diagnostics/package-info.java | 22 + .../CustomDelegationTokenManager.java | 70 + .../extensions/CustomTokenProviderAdaptee.java | 75 ++ .../fs/azurebfs/extensions/package-info.java | 32 + .../fs/azurebfs/oauth2/AccessTokenProvider.java | 98 ++ .../azurebfs/oauth2/AzureADAuthenticator.java | 344 +++++ .../hadoop/fs/azurebfs/oauth2/AzureADToken.java | 47 + .../oauth2/ClientCredsTokenProvider.java | 62 + .../oauth2/CustomTokenProviderAdapter.java | 58 + .../fs/azurebfs/oauth2/MsiTokenProvider.java | 48 + .../hadoop/fs/azurebfs/oauth2/QueryParams.java | 69 + .../oauth2/RefreshTokenBasedTokenProvider.java | 57 + .../oauth2/UserPasswordTokenProvider.java | 56 + .../hadoop/fs/azurebfs/oauth2/package-info.java | 18 + .../org/apache/hadoop/fs/azurebfs/package.html | 31 + .../security/AbfsDelegationTokenIdentifier.java | 49 + .../security/AbfsDelegationTokenManager.java | 88 ++ .../fs/azurebfs/security/AbfsTokenRenewer.java | 96 ++ .../fs/azurebfs/security/package-info.java | 23 + .../fs/azurebfs/services/AbfsAclHelper.java | 202 +++ .../hadoop/fs/azurebfs/services/AbfsClient.java | 581 ++++++++ .../services/AbfsClientThrottlingAnalyzer.java | 272 ++++ .../services/AbfsClientThrottlingIntercept.java | 135 ++ .../fs/azurebfs/services/AbfsHttpHeader.java | 40 + .../fs/azurebfs/services/AbfsHttpOperation.java | 446 ++++++ .../fs/azurebfs/services/AbfsInputStream.java | 381 ++++++ .../fs/azurebfs/services/AbfsOutputStream.java | 378 ++++++ .../fs/azurebfs/services/AbfsPermission.java | 114 ++ .../fs/azurebfs/services/AbfsRestOperation.java | 193 +++ .../services/AbfsRestOperationType.java | 42 + .../azurebfs/services/AbfsUriQueryBuilder.java | 64 + .../hadoop/fs/azurebfs/services/AuthType.java | 27 + .../services/ExponentialRetryPolicy.java | 144 ++ .../fs/azurebfs/services/KeyProvider.java | 43 + .../hadoop/fs/azurebfs/services/ReadBuffer.java | 139 ++ .../fs/azurebfs/services/ReadBufferManager.java | 395 ++++++ .../fs/azurebfs/services/ReadBufferWorker.java | 72 + .../azurebfs/services/SharedKeyCredentials.java | 510 +++++++ .../services/ShellDecryptionKeyProvider.java | 71 + .../fs/azurebfs/services/SimpleKeyProvider.java | 54 + .../fs/azurebfs/services/package-info.java | 22 + .../apache/hadoop/fs/azurebfs/utils/Base64.java | 329 +++++ .../fs/azurebfs/utils/SSLSocketFactoryEx.java | 240 ++++ .../hadoop/fs/azurebfs/utils/UriUtils.java | 78 ++ .../hadoop/fs/azurebfs/utils/package-info.java | 22 + ...apache.hadoop.security.token.TokenIdentifier | 1 + ...rg.apache.hadoop.security.token.TokenRenewer | 1 + .../hadoop-azure/src/site/markdown/abfs.md | 82 ++ .../src/site/markdown/testing_azure.md | 175 ++- .../fs/azure/AzureBlobStorageTestAccount.java | 22 +- ...TestFileSystemOperationExceptionMessage.java | 3 +- .../azure/ITestNativeFileSystemStatistics.java | 99 ++ .../fs/azure/ITestWasbUriAndConfiguration.java | 26 - .../fs/azure/NativeAzureFileSystemBaseTest.java | 80 +- .../azure/integration/AzureTestConstants.java | 6 +- .../fs/azure/integration/AzureTestUtils.java | 69 + .../azure/metrics/TestRollingWindowAverage.java | 4 +- .../azurebfs/AbstractAbfsIntegrationTest.java | 340 +++++ .../fs/azurebfs/AbstractAbfsScaleTest.java | 59 + .../azurebfs/AbstractAbfsTestWithTimeout.java | 70 + .../hadoop/fs/azurebfs/ITestAbfsClient.java | 49 + .../fs/azurebfs/ITestAbfsReadWriteAndSeek.java | 89 ++ .../ITestAzureBlobFileSystemAppend.java | 79 ++ .../ITestAzureBlobFileSystemBackCompat.java | 88 ++ .../azurebfs/ITestAzureBlobFileSystemCopy.java | 96 ++ .../ITestAzureBlobFileSystemCreate.java | 107 ++ .../ITestAzureBlobFileSystemDelete.java | 133 ++ .../azurebfs/ITestAzureBlobFileSystemE2E.java | 148 ++ .../ITestAzureBlobFileSystemE2EScale.java | 120 ++ .../ITestAzureBlobFileSystemFileStatus.java | 125 ++ .../ITestAzureBlobFileSystemFinalize.java | 64 + .../azurebfs/ITestAzureBlobFileSystemFlush.java | 387 ++++++ .../ITestAzureBlobFileSystemInitAndCreate.java | 53 + .../ITestAzureBlobFileSystemListStatus.java | 172 +++ .../azurebfs/ITestAzureBlobFileSystemMkDir.java | 48 + .../azurebfs/ITestAzureBlobFileSystemOauth.java | 178 +++ .../ITestAzureBlobFileSystemPermission.java | 108 ++ .../ITestAzureBlobFileSystemRandomRead.java | 588 ++++++++ .../ITestAzureBlobFileSystemRename.java | 152 +++ .../ITestAzureBlobFileSystemRenameUnicode.java | 98 ++ .../azurebfs/ITestAzureBlobFilesystemAcl.java | 1264 ++++++++++++++++++ .../azurebfs/ITestFileSystemInitialization.java | 77 ++ .../fs/azurebfs/ITestFileSystemProperties.java | 119 ++ .../azurebfs/ITestFileSystemRegistration.java | 113 ++ .../fs/azurebfs/ITestWasbAbfsCompatibility.java | 192 +++ .../TestAbfsConfigurationFieldsValidation.java | 179 +++ .../fs/azurebfs/TestAccountConfiguration.java | 285 ++++ .../constants/TestConfigurationKeys.java | 41 + .../fs/azurebfs/constants/package-info.java | 22 + .../contract/ABFSContractTestBinding.java | 67 + .../contract/AbfsFileSystemContract.java | 64 + .../ITestAbfsFileSystemContractAppend.java | 61 + .../ITestAbfsFileSystemContractConcat.java | 51 + .../ITestAbfsFileSystemContractCreate.java | 52 + .../ITestAbfsFileSystemContractDelete.java | 52 + .../ITestAbfsFileSystemContractDistCp.java | 49 + ...TestAbfsFileSystemContractGetFileStatus.java | 51 + .../ITestAbfsFileSystemContractMkdir.java | 52 + .../ITestAbfsFileSystemContractOpen.java | 52 + .../ITestAbfsFileSystemContractRename.java | 52 + ...TestAbfsFileSystemContractRootDirectory.java | 57 + ...ITestAbfsFileSystemContractSecureDistCp.java | 49 + .../ITestAbfsFileSystemContractSeek.java | 52 + .../ITestAbfsFileSystemContractSetTimes.java | 51 + .../ITestAzureBlobFileSystemBasics.java | 105 ++ .../fs/azurebfs/contract/package-info.java | 22 + .../TestConfigurationValidators.java | 121 ++ .../fs/azurebfs/diagnostics/package-info.java | 22 + .../apache/hadoop/fs/azurebfs/package-info.java | 22 + .../fs/azurebfs/services/TestAbfsClient.java | 86 ++ .../TestAbfsClientThrottlingAnalyzer.java | 177 +++ .../services/TestOauthFailOverHttp.java | 55 + .../fs/azurebfs/services/TestQueryParams.java | 72 + .../TestShellDecryptionKeyProvider.java | 92 ++ .../fs/azurebfs/services/package-info.java | 22 + .../hadoop/fs/azurebfs/utils/AbfsTestUtils.java | 85 ++ .../fs/azurebfs/utils/AclTestHelpers.java | 119 ++ .../hadoop/fs/azurebfs/utils/Parallelized.java | 60 + .../hadoop/fs/azurebfs/utils/TestUriUtils.java | 48 + .../hadoop/fs/azurebfs/utils/package-info.java | 22 + .../hadoop-azure/src/test/resources/abfs.xml | 64 + .../src/test/resources/azure-test.xml | 56 +- .../src/test/resources/log4j.properties | 34 + 181 files changed, 19711 insertions(+), 221 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0def6148/hadoop-project/pom.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
