Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5afc88d8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5afc88d8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5afc88d8 Branch: refs/heads/HDFS-12943 Commit: 5afc88d807366d8a8983fa526d92dc9d51ca7122 Parents: 741547e 8de5c92 Author: Erik Krogen <[email protected]> Authored: Mon Sep 24 11:37:24 2018 -0700 Committer: Erik Krogen <[email protected]> Committed: Mon Sep 24 11:37:24 2018 -0700 ---------------------------------------------------------------------- .gitignore | 1 + dev-support/bin/ozone-dist-layout-stitching | 181 --- dev-support/bin/ozone-dist-tar-stitching | 43 - .../java/org/apache/hadoop/fs/BBPartHandle.java | 5 + .../org/apache/hadoop/fs/BBUploadHandle.java | 5 + .../fs/CommonConfigurationKeysPublic.java | 4 +- .../hadoop/fs/FileSystemMultipartUploader.java | 4 + .../org/apache/hadoop/fs/MultipartUploader.java | 5 + .../hadoop/fs/MultipartUploaderFactory.java | 4 + .../src/main/resources/core-default.xml | 41 +- .../src/site/markdown/filesystem/filesystem.md | 11 +- .../conf/TestCommonConfigurationFields.java | 3 + .../apache/hadoop/conf/TestConfigRedactor.java | 7 + .../fs/contract/AbstractContractConcatTest.java | 34 +- .../AbstractContractGetFileStatusTest.java | 26 +- .../fs/contract/AbstractContractMkdirTest.java | 8 + .../hadoop/fs/contract/AbstractFSContract.java | 2 - .../hadoop/fs/contract/ContractTestUtils.java | 19 +- hadoop-dist/pom.xml | 118 -- hadoop-dist/src/main/compose/README.md | 51 - hadoop-dist/src/main/compose/ozone-hdfs/.env | 17 - .../main/compose/ozone-hdfs/docker-compose.yaml | 60 - .../src/main/compose/ozone-hdfs/docker-config | 76 -- hadoop-dist/src/main/compose/ozone/.env | 17 - .../src/main/compose/ozone/docker-compose.yaml | 49 - .../src/main/compose/ozone/docker-config | 75 -- .../main/compose/ozonefs/docker-compose.yaml | 59 - .../src/main/compose/ozonefs/docker-config | 35 - hadoop-dist/src/main/compose/ozoneperf/.env | 17 - .../src/main/compose/ozoneperf/README.md | 73 - .../src/main/compose/ozoneperf/compose-all.sh | 18 - .../compose/ozoneperf/docker-compose-freon.yaml | 26 - .../main/compose/ozoneperf/docker-compose.yaml | 64 - .../src/main/compose/ozoneperf/docker-config | 36 - hadoop-dist/src/main/compose/ozoneperf/init.sh | 21 - .../src/main/compose/ozoneperf/prometheus.yml | 24 - .../compose/ozonescripts/.ssh/authorized_keys | 16 - .../src/main/compose/ozonescripts/.ssh/config | 18 - .../main/compose/ozonescripts/.ssh/environment | 16 - .../src/main/compose/ozonescripts/.ssh/id_rsa | 42 - .../main/compose/ozonescripts/.ssh/id_rsa.pub | 16 - .../src/main/compose/ozonescripts/Dockerfile | 33 - .../src/main/compose/ozonescripts/README.md | 38 - .../compose/ozonescripts/docker-compose.yaml | 42 - .../src/main/compose/ozonescripts/docker-config | 38 - hadoop-dist/src/main/compose/ozonescripts/ps.sh | 17 - .../src/main/compose/ozonescripts/start.sh | 24 - .../src/main/compose/ozonescripts/stop.sh | 17 - hadoop-dist/src/main/ozone/README.txt | 51 - hadoop-dist/src/main/smoketest/README.md | 30 - .../src/main/smoketest/basic/basic.robot | 47 - .../src/main/smoketest/basic/ozone-shell.robot | 82 -- hadoop-dist/src/main/smoketest/commonlib.robot | 24 - .../src/main/smoketest/ozonefs/ozonefs.robot | 35 - hadoop-dist/src/main/smoketest/test.sh | 101 -- hadoop-hdds/client/pom.xml | 5 - hadoop-hdds/common/pom.xml | 2 - hadoop-hdds/container-service/pom.xml | 5 - hadoop-hdds/framework/pom.xml | 5 - hadoop-hdds/pom.xml | 30 + hadoop-hdds/server-scm/pom.xml | 50 +- .../src/main/resources/webapps/scm/index.html | 76 ++ .../src/main/resources/webapps/scm/main.html | 20 + .../resources/webapps/scm/scm-overview.html | 60 + .../src/main/resources/webapps/scm/scm.js | 54 + .../server-scm/src/main/webapps/scm/index.html | 76 -- .../server-scm/src/main/webapps/scm/main.html | 20 - .../src/main/webapps/scm/scm-overview.html | 60 - .../server-scm/src/main/webapps/scm/scm.js | 54 - hadoop-hdds/tools/pom.xml | 8 - .../hadoop/fs/http/client/HttpFSFileSystem.java | 17 + .../hadoop/fs/http/server/FSOperations.java | 73 + .../http/server/HttpFSParametersProvider.java | 2 + .../hadoop/fs/http/server/HttpFSServer.java | 16 + .../fs/http/client/BaseTestHttpFSWith.java | 132 +- .../hadoop/fs/http/server/TestHttpFSServer.java | 102 ++ .../TestDataNodeMultipleRegistrations.java | 26 +- hadoop-ozone/client/pom.xml | 6 - hadoop-ozone/common/pom.xml | 5 - hadoop-ozone/common/src/main/bin/ozone | 50 +- .../src/main/shellprofile.d/hadoop-ozone.sh | 24 - hadoop-ozone/datanode/pom.xml | 49 + .../dist/dev-support/bin/dist-layout-stitching | 113 ++ .../dist/dev-support/bin/dist-tar-stitching | 45 + hadoop-ozone/dist/pom.xml | 184 +++ hadoop-ozone/dist/src/main/compose/README.md | 51 + .../dist/src/main/compose/ozone-hdfs/.env | 17 + .../main/compose/ozone-hdfs/docker-compose.yaml | 60 + .../src/main/compose/ozone-hdfs/docker-config | 76 ++ hadoop-ozone/dist/src/main/compose/ozone/.env | 17 + .../src/main/compose/ozone/docker-compose.yaml | 49 + .../dist/src/main/compose/ozone/docker-config | 75 ++ .../main/compose/ozonefs/docker-compose.yaml | 59 + .../dist/src/main/compose/ozonefs/docker-config | 35 + .../dist/src/main/compose/ozoneperf/.env | 17 + .../dist/src/main/compose/ozoneperf/README.md | 73 + .../src/main/compose/ozoneperf/compose-all.sh | 18 + .../compose/ozoneperf/docker-compose-freon.yaml | 26 + .../main/compose/ozoneperf/docker-compose.yaml | 64 + .../src/main/compose/ozoneperf/docker-config | 35 + .../dist/src/main/compose/ozoneperf/init.sh | 21 + .../src/main/compose/ozoneperf/prometheus.yml | 24 + .../compose/ozonescripts/.ssh/authorized_keys | 16 + .../src/main/compose/ozonescripts/.ssh/config | 18 + .../main/compose/ozonescripts/.ssh/environment | 16 + .../src/main/compose/ozonescripts/.ssh/id_rsa | 42 + .../main/compose/ozonescripts/.ssh/id_rsa.pub | 16 + .../src/main/compose/ozonescripts/Dockerfile | 33 + .../src/main/compose/ozonescripts/README.md | 38 + .../compose/ozonescripts/docker-compose.yaml | 42 + .../src/main/compose/ozonescripts/docker-config | 38 + .../dist/src/main/compose/ozonescripts/ps.sh | 17 + .../dist/src/main/compose/ozonescripts/start.sh | 24 + .../dist/src/main/compose/ozonescripts/stop.sh | 17 + hadoop-ozone/dist/src/main/ozone/README.txt | 51 + hadoop-ozone/dist/src/main/smoketest/README.md | 30 + .../dist/src/main/smoketest/basic/basic.robot | 47 + .../src/main/smoketest/basic/ozone-shell.robot | 82 ++ .../dist/src/main/smoketest/commonlib.robot | 24 + .../src/main/smoketest/ozonefs/ozonefs.robot | 35 + hadoop-ozone/dist/src/main/smoketest/test.sh | 101 ++ hadoop-ozone/docs/pom.xml | 5 - hadoop-ozone/integration-test/pom.xml | 14 - .../hadoop/ozone/freon/TestDataValidate.java | 115 -- .../ozone/freon/TestRandomKeyGenerator.java | 106 -- .../apache/hadoop/ozone/om/TestOmSQLCli.java | 285 ---- .../hadoop/ozone/scm/TestContainerSQLCli.java | 253 ---- hadoop-ozone/objectstore-service/pom.xml | 5 - hadoop-ozone/ozone-manager/pom.xml | 50 +- .../resources/webapps/ozoneManager/index.html | 70 + .../resources/webapps/ozoneManager/main.css | 23 + .../resources/webapps/ozoneManager/main.html | 18 + .../webapps/ozoneManager/om-metrics.html | 44 + .../webapps/ozoneManager/ozoneManager.js | 110 ++ .../src/main/webapps/ozoneManager/index.html | 70 - .../src/main/webapps/ozoneManager/main.css | 23 - .../src/main/webapps/ozoneManager/main.html | 18 - .../main/webapps/ozoneManager/om-metrics.html | 44 - .../main/webapps/ozoneManager/ozoneManager.js | 110 -- hadoop-ozone/ozonefs/pom.xml | 2 - hadoop-ozone/pom.xml | 25 +- hadoop-ozone/tools/pom.xml | 32 +- .../hadoop/ozone/freon/TestDataValidate.java | 115 ++ .../ozone/freon/TestRandomKeyGenerator.java | 106 ++ .../apache/hadoop/ozone/om/TestOmSQLCli.java | 285 ++++ .../hadoop/ozone/scm/TestContainerSQLCli.java | 253 ++++ hadoop-project/pom.xml | 23 + .../hadoop/fs/s3a/S3AMultipartUploader.java | 4 + hadoop-tools/hadoop-azure/pom.xml | 403 +++++- .../src/config/checkstyle-suppressions.xml | 49 + .../fs/azure/ClientThrottlingAnalyzer.java | 2 +- .../org/apache/hadoop/fs/azurebfs/Abfs.java | 46 + .../hadoop/fs/azurebfs/AbfsConfiguration.java | 576 ++++++++ .../org/apache/hadoop/fs/azurebfs/Abfss.java | 46 + .../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 953 +++++++++++++ .../fs/azurebfs/AzureBlobFileSystemStore.java | 1028 ++++++++++++++ .../fs/azurebfs/SecureAzureBlobFileSystem.java | 39 + .../azurebfs/constants/AbfsHttpConstants.java | 91 ++ .../azurebfs/constants/ConfigurationKeys.java | 89 ++ .../constants/FileSystemConfigurations.java | 67 + .../constants/FileSystemUriSchemes.java | 42 + .../constants/HttpHeaderConfigurations.java | 63 + .../fs/azurebfs/constants/HttpQueryParams.java | 40 + .../fs/azurebfs/constants/package-info.java | 22 + .../ConfigurationValidationAnnotations.java | 104 ++ .../contracts/annotations/package-info.java | 22 + .../diagnostics/ConfigurationValidator.java | 37 + .../contracts/diagnostics/package-info.java | 22 + .../exceptions/AbfsRestOperationException.java | 84 ++ .../AzureBlobFileSystemException.java | 56 + .../ConfigurationPropertyNotFoundException.java | 32 + .../FileSystemOperationUnhandledException.java | 33 + .../InvalidAbfsRestOperationException.java | 40 + .../InvalidAclOperationException.java | 33 + .../InvalidConfigurationValueException.java | 37 + .../InvalidFileSystemPropertyException.java | 33 + .../InvalidUriAuthorityException.java | 33 + .../exceptions/InvalidUriException.java | 33 + .../exceptions/KeyProviderException.java | 42 + .../contracts/exceptions/TimeoutException.java | 33 + .../TokenAccessProviderException.java | 36 + .../contracts/exceptions/package-info.java | 22 + .../fs/azurebfs/contracts/package-info.java | 22 + .../services/AzureServiceErrorCode.java | 115 ++ .../services/ListResultEntrySchema.java | 239 ++++ .../contracts/services/ListResultSchema.java | 58 + .../contracts/services/ReadBufferStatus.java | 29 + .../contracts/services/package-info.java | 22 + ...Base64StringConfigurationBasicValidator.java | 50 + .../BooleanConfigurationBasicValidator.java | 50 + .../ConfigurationBasicValidator.java | 67 + .../IntegerConfigurationBasicValidator.java | 68 + .../LongConfigurationBasicValidator.java | 63 + .../StringConfigurationBasicValidator.java | 43 + .../fs/azurebfs/diagnostics/package-info.java | 22 + .../CustomDelegationTokenManager.java | 70 + .../extensions/CustomTokenProviderAdaptee.java | 75 ++ .../fs/azurebfs/extensions/package-info.java | 32 + .../fs/azurebfs/oauth2/AccessTokenProvider.java | 98 ++ .../azurebfs/oauth2/AzureADAuthenticator.java | 344 +++++ .../hadoop/fs/azurebfs/oauth2/AzureADToken.java | 47 + .../oauth2/ClientCredsTokenProvider.java | 62 + .../oauth2/CustomTokenProviderAdapter.java | 58 + .../fs/azurebfs/oauth2/MsiTokenProvider.java | 48 + .../hadoop/fs/azurebfs/oauth2/QueryParams.java | 69 + .../oauth2/RefreshTokenBasedTokenProvider.java | 57 + .../oauth2/UserPasswordTokenProvider.java | 56 + .../hadoop/fs/azurebfs/oauth2/package-info.java | 18 + .../org/apache/hadoop/fs/azurebfs/package.html | 31 + .../security/AbfsDelegationTokenIdentifier.java | 49 + .../security/AbfsDelegationTokenManager.java | 88 ++ .../fs/azurebfs/security/AbfsTokenRenewer.java | 96 ++ .../fs/azurebfs/security/package-info.java | 23 + .../fs/azurebfs/services/AbfsAclHelper.java | 202 +++ .../hadoop/fs/azurebfs/services/AbfsClient.java | 581 ++++++++ .../services/AbfsClientThrottlingAnalyzer.java | 272 ++++ .../services/AbfsClientThrottlingIntercept.java | 135 ++ .../fs/azurebfs/services/AbfsHttpHeader.java | 40 + .../fs/azurebfs/services/AbfsHttpOperation.java | 446 ++++++ .../fs/azurebfs/services/AbfsInputStream.java | 381 ++++++ .../fs/azurebfs/services/AbfsOutputStream.java | 378 ++++++ .../fs/azurebfs/services/AbfsPermission.java | 114 ++ .../fs/azurebfs/services/AbfsRestOperation.java | 193 +++ .../services/AbfsRestOperationType.java | 42 + .../azurebfs/services/AbfsUriQueryBuilder.java | 64 + .../hadoop/fs/azurebfs/services/AuthType.java | 27 + .../services/ExponentialRetryPolicy.java | 144 ++ .../fs/azurebfs/services/KeyProvider.java | 43 + .../hadoop/fs/azurebfs/services/ReadBuffer.java | 139 ++ .../fs/azurebfs/services/ReadBufferManager.java | 395 ++++++ .../fs/azurebfs/services/ReadBufferWorker.java | 72 + .../azurebfs/services/SharedKeyCredentials.java | 510 +++++++ .../services/ShellDecryptionKeyProvider.java | 71 + .../fs/azurebfs/services/SimpleKeyProvider.java | 54 + .../fs/azurebfs/services/package-info.java | 22 + .../apache/hadoop/fs/azurebfs/utils/Base64.java | 329 +++++ .../fs/azurebfs/utils/SSLSocketFactoryEx.java | 240 ++++ .../hadoop/fs/azurebfs/utils/UriUtils.java | 78 ++ .../hadoop/fs/azurebfs/utils/package-info.java | 22 + ...apache.hadoop.security.token.TokenIdentifier | 1 + ...rg.apache.hadoop.security.token.TokenRenewer | 1 + .../hadoop-azure/src/site/markdown/abfs.md | 82 ++ .../src/site/markdown/testing_azure.md | 175 ++- .../fs/azure/AzureBlobStorageTestAccount.java | 22 +- ...TestFileSystemOperationExceptionMessage.java | 3 +- .../azure/ITestNativeFileSystemStatistics.java | 99 ++ .../fs/azure/ITestWasbUriAndConfiguration.java | 26 - .../fs/azure/NativeAzureFileSystemBaseTest.java | 80 +- .../azure/integration/AzureTestConstants.java | 6 +- .../fs/azure/integration/AzureTestUtils.java | 69 + .../azure/metrics/TestRollingWindowAverage.java | 4 +- .../azurebfs/AbstractAbfsIntegrationTest.java | 340 +++++ .../fs/azurebfs/AbstractAbfsScaleTest.java | 59 + .../azurebfs/AbstractAbfsTestWithTimeout.java | 70 + .../hadoop/fs/azurebfs/ITestAbfsClient.java | 49 + .../fs/azurebfs/ITestAbfsReadWriteAndSeek.java | 89 ++ .../ITestAzureBlobFileSystemAppend.java | 79 ++ .../ITestAzureBlobFileSystemBackCompat.java | 88 ++ .../azurebfs/ITestAzureBlobFileSystemCopy.java | 96 ++ .../ITestAzureBlobFileSystemCreate.java | 107 ++ .../ITestAzureBlobFileSystemDelete.java | 133 ++ .../azurebfs/ITestAzureBlobFileSystemE2E.java | 148 ++ .../ITestAzureBlobFileSystemE2EScale.java | 120 ++ .../ITestAzureBlobFileSystemFileStatus.java | 125 ++ .../ITestAzureBlobFileSystemFinalize.java | 64 + .../azurebfs/ITestAzureBlobFileSystemFlush.java | 387 ++++++ .../ITestAzureBlobFileSystemInitAndCreate.java | 53 + .../ITestAzureBlobFileSystemListStatus.java | 172 +++ .../azurebfs/ITestAzureBlobFileSystemMkDir.java | 48 + .../azurebfs/ITestAzureBlobFileSystemOauth.java | 178 +++ .../ITestAzureBlobFileSystemPermission.java | 108 ++ .../ITestAzureBlobFileSystemRandomRead.java | 588 ++++++++ .../ITestAzureBlobFileSystemRename.java | 152 +++ .../ITestAzureBlobFileSystemRenameUnicode.java | 98 ++ .../azurebfs/ITestAzureBlobFilesystemAcl.java | 1264 ++++++++++++++++++ .../azurebfs/ITestFileSystemInitialization.java | 77 ++ .../fs/azurebfs/ITestFileSystemProperties.java | 119 ++ .../azurebfs/ITestFileSystemRegistration.java | 113 ++ .../fs/azurebfs/ITestWasbAbfsCompatibility.java | 192 +++ .../TestAbfsConfigurationFieldsValidation.java | 179 +++ .../fs/azurebfs/TestAccountConfiguration.java | 285 ++++ .../constants/TestConfigurationKeys.java | 41 + .../fs/azurebfs/constants/package-info.java | 22 + .../contract/ABFSContractTestBinding.java | 67 + .../contract/AbfsFileSystemContract.java | 64 + .../ITestAbfsFileSystemContractAppend.java | 61 + .../ITestAbfsFileSystemContractConcat.java | 51 + .../ITestAbfsFileSystemContractCreate.java | 52 + .../ITestAbfsFileSystemContractDelete.java | 52 + .../ITestAbfsFileSystemContractDistCp.java | 49 + ...TestAbfsFileSystemContractGetFileStatus.java | 51 + .../ITestAbfsFileSystemContractMkdir.java | 52 + .../ITestAbfsFileSystemContractOpen.java | 52 + .../ITestAbfsFileSystemContractRename.java | 52 + ...TestAbfsFileSystemContractRootDirectory.java | 57 + ...ITestAbfsFileSystemContractSecureDistCp.java | 49 + .../ITestAbfsFileSystemContractSeek.java | 52 + .../ITestAbfsFileSystemContractSetTimes.java | 51 + .../ITestAzureBlobFileSystemBasics.java | 105 ++ .../fs/azurebfs/contract/package-info.java | 22 + .../TestConfigurationValidators.java | 121 ++ .../fs/azurebfs/diagnostics/package-info.java | 22 + .../apache/hadoop/fs/azurebfs/package-info.java | 22 + .../fs/azurebfs/services/TestAbfsClient.java | 86 ++ .../TestAbfsClientThrottlingAnalyzer.java | 177 +++ .../services/TestOauthFailOverHttp.java | 55 + .../fs/azurebfs/services/TestQueryParams.java | 72 + .../TestShellDecryptionKeyProvider.java | 92 ++ .../fs/azurebfs/services/package-info.java | 22 + .../hadoop/fs/azurebfs/utils/AbfsTestUtils.java | 85 ++ .../fs/azurebfs/utils/AclTestHelpers.java | 119 ++ .../hadoop/fs/azurebfs/utils/Parallelized.java | 60 + .../hadoop/fs/azurebfs/utils/TestUriUtils.java | 48 + .../hadoop/fs/azurebfs/utils/package-info.java | 22 + .../hadoop-azure/src/test/resources/abfs.xml | 64 + .../src/test/resources/azure-test.xml | 56 +- .../src/test/resources/log4j.properties | 34 + .../webapp/app/controllers/yarn-app/logs.js | 7 +- .../src/main/webapp/app/styles/yarn-app.scss | 5 + .../src/main/webapp/app/templates/yarn-app.hbs | 6 +- .../main/webapp/app/templates/yarn-app/logs.hbs | 49 +- 321 files changed, 23322 insertions(+), 3410 deletions(-) ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
