Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b5f772ab Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b5f772ab Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b5f772ab Branch: refs/heads/HDFS-7240 Commit: b5f772abdae63d11cc7d9cc60e0892e2b95bef44 Parents: 2392308 4aefe11 Author: Anu Engineer <aengin...@apache.org> Authored: Thu Jun 16 11:29:40 2016 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Thu Jun 16 11:29:40 2016 -0700 ---------------------------------------------------------------------- LICENSE.txt | 40 +- NOTICE.txt | 2 +- dev-support/bin/create-release | 69 +- dev-support/docker/Dockerfile | 5 + .../classification/tools/RootDocProcessor.java | 4 + .../authentication/client/AuthenticatedURL.java | 4 +- .../hadoop-auth/src/site/markdown/Examples.md | 4 +- hadoop-common-project/hadoop-common/pom.xml | 3 +- .../java/org/apache/hadoop/fs/FileUtil.java | 14 + .../java/org/apache/hadoop/fs/shell/Ls.java | 28 +- .../apache/hadoop/fs/shell/PrintableString.java | 72 + .../apache/hadoop/io/compress/BZip2Codec.java | 7 +- .../hadoop/io/retry/RetryInvocationHandler.java | 12 +- .../main/java/org/apache/hadoop/ipc/Server.java | 2 +- .../hadoop/security/LdapGroupsMapping.java | 114 +- .../hadoop/util/GenericOptionsParser.java | 2 +- .../src/main/resources/core-default.xml | 13 + .../src/site/markdown/FileSystemShell.md | 3 +- .../org/apache/hadoop/fs/TestFsShellList.java | 78 + .../org/apache/hadoop/fs/TestGetSpaceUsed.java | 5 +- .../hadoop/fs/shell/TestPrintableString.java | 87 + .../apache/hadoop/fs/shell/package-info.java | 26 + .../apache/hadoop/io/retry/TestRetryProxy.java | 7 +- .../java/org/apache/hadoop/ipc/TestRPC.java | 4 + .../sink/TestRollingFileSystemSink.java | 8 +- .../hadoop/security/TestLdapGroupsMapping.java | 62 +- .../security/TestLdapGroupsMappingBase.java | 33 +- .../TestLdapGroupsMappingWithPosixGroup.java | 2 +- .../hadoop/security/token/TestDtUtilShell.java | 3 +- .../src/test/resources/testConf.xml | 6 +- .../hdfs/shortcircuit/TestShortCircuitShm.java | 4 +- .../TestHttpFSFWithSWebhdfsFileSystem.java | 5 +- .../TestHttpFSFileSystemLocalFileSystem.java | 5 +- .../hdfs/nfs/nfs3/TestNfs3HttpServer.java | 5 +- .../BlockPlacementStatusWithUpgradeDomain.java | 2 +- .../CacheReplicationMonitor.java | 8 + .../server/namenode/EncryptionZoneManager.java | 2 +- .../server/namenode/FSDirEncryptionZoneOp.java | 3 + .../server/namenode/FSImageFormatPBINode.java | 10 +- .../hdfs/server/namenode/NamenodeFsck.java | 24 +- .../org/apache/hadoop/hdfs/tools/DFSck.java | 16 +- .../OfflineImageReconstructor.java | 30 +- .../src/main/webapps/hdfs/explorer.html | 9 + .../src/main/webapps/hdfs/explorer.js | 83 +- .../src/main/webapps/static/hadoop.css | 7 + .../src/site/markdown/HDFSCommands.md | 3 +- .../src/site/markdown/HdfsRollingUpgrade.md | 2 +- .../apache/hadoop/cli/TestCryptoAdminCLI.java | 4 +- .../hadoop/fs/TestEnhancedByteBufferAccess.java | 12 +- .../TestSWebHdfsFileContextMainOperations.java | 6 +- .../TestWebHdfsFileContextMainOperations.java | 7 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 17 +- .../hadoop/hdfs/TestAppendSnapshotTruncate.java | 4 +- .../org/apache/hadoop/hdfs/TestDFSShell.java | 4 +- .../hadoop/hdfs/TestDFSUpgradeFromImage.java | 18 +- .../org/apache/hadoop/hdfs/TestDFSUtil.java | 3 +- .../hadoop/hdfs/TestDatanodeLayoutUpgrade.java | 21 +- ...estDatanodeStartupFixesLegacyStorageIDs.java | 12 +- .../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 7 +- .../org/apache/hadoop/hdfs/TestFetchImage.java | 5 +- .../java/org/apache/hadoop/hdfs/TestPread.java | 2 +- .../org/apache/hadoop/hdfs/TestSeekBug.java | 3 +- .../sasl/SaslDataTransferTestCase.java | 5 +- .../hdfs/qjournal/TestSecureNNWithQJM.java | 5 +- .../hdfs/server/balancer/TestBalancer.java | 3 +- ...stBlockPlacementStatusWithUpgradeDomain.java | 83 + .../hdfs/server/datanode/TestDataNodeUUID.java | 3 +- .../hdfs/server/datanode/TestDataStorage.java | 3 +- .../server/namenode/TestCreateEditsLog.java | 6 +- .../namenode/TestDecommissioningStatus.java | 65 +- .../namenode/TestEditLogFileInputStream.java | 3 +- .../namenode/TestFSImageWithSnapshot.java | 4 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 85 +- .../server/namenode/TestNameNodeHttpServer.java | 5 +- .../server/namenode/TestNameNodeRecovery.java | 5 +- .../TestNameNodeRespectsBindHostKeys.java | 5 +- .../TestUpgradeDomainBlockPlacementPolicy.java | 161 +- .../snapshot/TestRenameWithSnapshots.java | 2 +- .../server/namenode/snapshot/TestSnapshot.java | 4 +- .../apache/hadoop/hdfs/tools/TestGetConf.java | 62 +- .../TestOfflineImageViewer.java | 19 + .../hadoop/hdfs/util/HostsFileWriter.java | 32 +- .../hdfs/util/TestCombinedHostsFileReader.java | 5 +- .../hadoop/hdfs/web/TestHttpsFileSystem.java | 5 +- .../hadoop/hdfs/web/TestWebHdfsTokens.java | 9 +- .../java/org/apache/hadoop/test/PathUtils.java | 7 +- .../org/apache/hadoop/mapred/YarnChild.java | 10 +- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 2 +- .../v2/app/job/impl/TaskAttemptImpl.java | 7 +- .../v2/app/webapp/dao/TaskAttemptInfo.java | 4 +- .../apache/hadoop/mapreduce/v2/app/MRApp.java | 2 +- .../mapreduce/v2/app/TestMRAppMaster.java | 66 +- .../app/commit/TestCommitterEventHandler.java | 12 +- .../mapreduce/v2/app/job/impl/TestJobImpl.java | 4 +- .../app/webapp/TestAMWebServicesAttempts.java | 2 +- .../mapred/LocalDistributedCacheManager.java | 2 +- .../apache/hadoop/mapreduce/v2/util/MRApps.java | 8 +- .../mapreduce/jobhistory/AMStartedEvent.java | 6 +- .../jobhistory/TaskAttemptStartedEvent.java | 4 +- .../lib/input/UncompressedSplitLineReader.java | 8 +- .../v2/hs/webapp/TestHsWebServicesAttempts.java | 2 +- .../org/apache/hadoop/mapred/YARNRunner.java | 5 +- .../hadoop/mapred/TestTextInputFormat.java | 108 +- .../apache/hadoop/mapreduce/v2/TestMRJobs.java | 2 +- .../apache/hadoop/mapred/ShuffleHandler.java | 2 +- .../apache/hadoop/tools/HadoopArchiveLogs.java | 3 +- .../hadoop-aws/dev-support/findbugs-exclude.xml | 7 + .../org/apache/hadoop/fs/s3/S3Credentials.java | 31 +- .../org/apache/hadoop/fs/s3/S3FileSystem.java | 20 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 79 +- .../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 46 + .../hadoop/fs/s3native/NativeS3FileSystem.java | 19 +- .../hadoop/fs/s3native/S3xLoginHelper.java | 283 + .../src/site/markdown/tools/hadoop-aws/index.md | 5 +- .../apache/hadoop/fs/s3/TestS3FileSystem.java | 24 +- .../hadoop/fs/s3a/TestS3AConfiguration.java | 55 +- .../hadoop/fs/s3a/TestS3ACredentialsInURL.java | 153 + .../hadoop/fs/s3native/TestS3xLoginHelper.java | 197 + .../web/PrivateAzureDataLakeFileSystem.java | 2 +- .../hadoop/fs/adl/TestADLResponseData.java | 120 + .../apache/hadoop/fs/adl/TestGetFileStatus.java | 65 + .../apache/hadoop/fs/adl/TestListStatus.java | 95 + .../hadoop/fs/adl/TestableAdlFileSystem.java | 31 + .../fs/adl/live/AdlStorageConfiguration.java | 76 + .../hadoop/fs/adl/live/AdlStorageContract.java | 65 + .../fs/adl/live/TestAdlContractAppendLive.java | 53 + .../fs/adl/live/TestAdlContractConcatLive.java | 52 + .../fs/adl/live/TestAdlContractCreateLive.java | 52 + .../fs/adl/live/TestAdlContractDeleteLive.java | 44 + .../fs/adl/live/TestAdlContractMkdirLive.java | 55 + .../fs/adl/live/TestAdlContractOpenLive.java | 44 + .../fs/adl/live/TestAdlContractRenameLive.java | 63 + .../fs/adl/live/TestAdlContractRootDirLive.java | 52 + .../fs/adl/live/TestAdlContractSeekLive.java | 44 + .../live/TestAdlDifferentSizeWritesLive.java | 102 + .../adl/live/TestAdlFileSystemContractLive.java | 111 + .../hadoop/fs/adl/live/TestAdlReadLive.java | 342 + ...estAdlWebHdfsFileContextCreateMkdirLive.java | 79 + ...AdlWebHdfsFileContextMainOperationsLive.java | 104 + ...hedRefreshTokenBasedAccessTokenProvider.java | 8 +- .../hadoop/fs/common/AdlMockWebServer.java | 116 + .../hadoop/fs/common/ExpectedResponse.java | 72 + .../hadoop/fs/common/TestDataForRead.java | 120 + .../org/apache/hadoop/hdfs/web/TestAdlRead.java | 205 + .../web/TestConcurrentDataReadOperations.java | 306 + .../hdfs/web/TestConfigurationSetting.java | 112 + .../hdfs/web/TestSplitSizeCalculation.java | 123 + .../src/test/resources/adls.xml | 139 + .../test/resources/contract-test-options.xml | 57 + .../jdiff/Apache_Hadoop_YARN_API_2.6.0.xml | 13076 ++++++++++++++++ .../jdiff/Apache_Hadoop_YARN_API_2.7.2.xml | 13692 +++++++++++++++++ .../jdiff/Apache_Hadoop_YARN_Client.2.6.0.xml | 2427 +++ .../jdiff/Apache_Hadoop_YARN_Client_2.7.2.xml | 2581 ++++ .../jdiff/Apache_Hadoop_YARN_Common_2.6.0.xml | 2870 ++++ .../jdiff/Apache_Hadoop_YARN_Common_2.7.2.xml | 3323 ++++ .../Apache_Hadoop_YARN_Server_Common_2.6.0.xml | 2059 +++ .../Apache_Hadoop_YARN_Server_Common_2.7.2.xml | 1801 +++ .../hadoop-yarn/dev-support/jdiff/Null.java | 20 + .../hadoop-yarn/hadoop-yarn-api/pom.xml | 2 + .../yarn/api/records/ApplicationAttemptId.java | 36 +- .../hadoop/yarn/api/records/ApplicationId.java | 38 +- .../hadoop/yarn/api/records/Container.java | 45 + .../hadoop/yarn/api/records/ContainerId.java | 6 +- .../apache/hadoop/yarn/api/records/NodeId.java | 23 +- .../yarn/api/records/ResourceRequest.java | 52 + .../org/apache/hadoop/yarn/api/records/URL.java | 49 + .../src/main/proto/yarn_protos.proto | 2 + .../distributedshell/ApplicationMaster.java | 7 +- .../applications/distributedshell/Client.java | 3 +- .../DistributedShellTimelinePlugin.java | 4 +- .../distributedshell/TestDistributedShell.java | 4 +- .../hadoop-yarn/hadoop-yarn-client/pom.xml | 2 + .../hadoop/yarn/client/api/AMRMClient.java | 43 +- .../yarn/client/api/async/AMRMClientAsync.java | 17 + .../yarn/client/api/impl/AMRMClientImpl.java | 294 +- .../client/api/impl/RemoteRequestsTable.java | 332 + .../hadoop/yarn/client/cli/ApplicationCLI.java | 27 +- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 20 +- .../apache/hadoop/yarn/client/cli/NodeCLI.java | 2 +- .../hadoop/yarn/client/cli/RMAdminCLI.java | 2 +- .../client/api/impl/BaseAMRMProxyE2ETest.java | 197 + .../yarn/client/api/impl/TestAMRMClient.java | 26 +- .../impl/TestAMRMClientContainerRequest.java | 54 +- .../api/impl/TestAMRMClientOnRMRestart.java | 16 +- .../yarn/client/api/impl/TestAMRMProxy.java | 171 +- .../api/impl/TestDistributedScheduling.java | 644 +- .../yarn/client/api/impl/TestNMClient.java | 7 +- .../hadoop/yarn/client/cli/TestLogsCLI.java | 25 + .../hadoop/yarn/client/cli/TestRMAdminCLI.java | 2 +- .../src/test/resources/core-site.xml | 25 - .../hadoop-yarn/hadoop-yarn-common/pom.xml | 2 + .../api/records/impl/pb/ContainerPBImpl.java | 14 + .../records/impl/pb/ResourceRequestPBImpl.java | 21 +- .../AggregatedLogDeletionService.java | 2 +- .../logaggregation/AggregatedLogFormat.java | 59 +- .../yarn/logaggregation/LogCLIHelpers.java | 7 +- .../apache/hadoop/yarn/util/ConverterUtils.java | 206 +- .../org/apache/hadoop/yarn/util/FSDownload.java | 2 +- .../yarn/webapp/log/AggregatedLogsBlock.java | 4 +- .../hadoop/yarn/webapp/util/WebAppUtils.java | 2 +- .../hadoop/yarn/util/TestConverterUtils.java | 30 +- .../apache/hadoop/yarn/util/TestFSDownload.java | 13 +- ...pplicationHistoryManagerOnTimelineStore.java | 25 +- .../FileSystemApplicationHistoryStore.java | 8 +- .../hadoop-yarn-server-common/pom.xml | 2 + .../hadoop/yarn/server/utils/BuilderUtils.java | 2 +- .../yarn/server/webapp/AppAttemptBlock.java | 2 +- .../yarn/server/webapp/ContainerBlock.java | 2 +- .../hadoop/yarn/server/webapp/WebServices.java | 6 +- .../nodemanager/DefaultContainerExecutor.java | 5 +- .../nodemanager/DockerContainerExecutor.java | 6 +- .../nodemanager/LinuxContainerExecutor.java | 2 +- .../container/ContainerImpl.java | 2 +- .../launcher/ContainerLaunch.java | 10 +- .../launcher/ContainerRelaunch.java | 2 +- .../launcher/RecoveredContainerLaunch.java | 6 +- .../localizer/ContainerLocalizer.java | 3 +- .../localizer/LocalResourceRequest.java | 4 +- .../localizer/ResourceLocalizationService.java | 28 +- .../event/LocalizerResourceRequestEvent.java | 2 +- .../sharedcache/SharedCacheUploader.java | 2 +- .../logaggregation/AppLogAggregatorImpl.java | 10 +- .../recovery/NMLeveldbStateStoreService.java | 10 +- .../util/NodeManagerBuilderUtils.java | 2 +- .../nodemanager/util/ProcessIdFileReader.java | 3 +- .../nodemanager/webapp/ApplicationPage.java | 5 +- .../nodemanager/webapp/ContainerLogsPage.java | 2 +- .../nodemanager/webapp/ContainerLogsUtils.java | 6 +- .../nodemanager/webapp/ContainerPage.java | 2 +- .../nodemanager/webapp/NMWebServices.java | 4 +- .../server/nodemanager/webapp/dao/AppInfo.java | 4 +- .../nodemanager/TestNodeManagerReboot.java | 2 +- .../nodemanager/TestNodeManagerResync.java | 2 +- .../nodemanager/TestNodeManagerShutdown.java | 2 +- .../impl/pb/TestPBRecordImpl.java | 9 +- .../containermanager/TestContainerManager.java | 22 +- .../TestContainerManagerRecovery.java | 2 +- .../TestContainerManagerRegression.java | 84 + .../launcher/TestContainerLaunch.java | 8 +- .../localizer/TestContainerLocalizer.java | 2 +- .../localizer/TestLocalResource.java | 7 +- .../TestResourceLocalizationService.java | 21 +- .../TestAppLogAggregatorImpl.java | 14 +- .../TestLogAggregationService.java | 136 +- .../monitor/TestContainersMonitor.java | 2 +- .../TestNMLeveldbStateStoreService.java | 24 +- .../nodemanager/webapp/TestNMWebServer.java | 2 +- .../webapp/TestNMWebServicesContainers.java | 7 +- .../server/resourcemanager/ResourceManager.java | 2 +- .../recovery/LeveldbRMStateStore.java | 5 +- .../recovery/ZKRMStateStore.java | 2 +- .../resource/DynamicResourceConfiguration.java | 2 +- .../rmcontainer/RMContainerImpl.java | 2 +- .../resourcemanager/rmnode/RMNodeImpl.java | 37 + .../scheduler/fair/FSAppAttempt.java | 62 +- .../resourcemanager/webapp/RMAppsBlock.java | 4 +- .../resourcemanager/webapp/RMWebAppFilter.java | 2 +- .../resourcemanager/webapp/RMWebServices.java | 9 +- .../webapp/dao/AppAttemptInfo.java | 2 +- .../resourcemanager/webapp/dao/AppInfo.java | 3 +- .../yarn/server/resourcemanager/MockNM.java | 57 +- .../resourcemanager/TestAMAuthorization.java | 22 + .../resourcemanager/TestClientRMTokens.java | 22 + .../resourcemanager/TestRMAdminService.java | 8 +- .../resourcemanager/TestRMNodeTransitions.java | 44 + .../TestWorkPreservingRMRestart.java | 2 +- .../recovery/RMStateStoreTestBase.java | 14 +- .../recovery/TestFSRMStateStore.java | 4 +- .../recovery/TestZKRMStateStore.java | 11 +- .../scheduler/fair/TestFairScheduler.java | 72 + .../TestRMWebServicesAppsModification.java | 7 +- ...ebServicesDelegationTokenAuthentication.java | 5 +- .../webapp/TestRMWebappAuthentication.java | 9 +- .../timeline/EntityGroupFSTimelineStore.java | 2 +- .../timeline/EntityGroupPlugInForTest.java | 7 +- .../TestEntityGroupFSTimelineStore.java | 4 +- hadoop-yarn-project/hadoop-yarn/pom.xml | 129 + pom.xml | 4 +- 278 files changed, 49457 insertions(+), 1734 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5f772ab/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org