Merge branch 'trunk' into HDFS-1312
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76a1391d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76a1391d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76a1391d Branch: refs/heads/HDFS-1312 Commit: 76a1391d53373aa99251f669aab35d9e54e866f4 Parents: 7c79136 c14c1b2 Author: Anu Engineer <aengin...@apache.org> Authored: Tue Jun 7 10:31:57 2016 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Tue Jun 7 10:31:57 2016 -0700 ---------------------------------------------------------------------- .../KeyProviderDelegationTokenExtension.java | 30 +- .../crypto/key/kms/KMSClientProvider.java | 158 +++++- .../key/kms/LoadBalancingKMSClientProvider.java | 21 + .../hadoop/crypto/key/kms/ValueQueue.java | 2 +- .../org/apache/hadoop/fs/LocalDirAllocator.java | 153 +++--- .../hadoop/metrics2/impl/MetricsSystemImpl.java | 1 + .../metrics2/lib/DefaultMetricsSystem.java | 9 + .../hadoop/security/LdapGroupsMapping.java | 12 + .../hadoop/security/token/DtFileOperations.java | 23 + .../hadoop/security/token/DtUtilShell.java | 47 +- .../DelegationTokenAuthenticationHandler.java | 7 + .../web/DelegationTokenAuthenticator.java | 19 + .../java/org/apache/hadoop/util/KMSUtil.java | 76 +++ ...rg.apache.hadoop.security.token.TokenRenewer | 14 + .../src/main/resources/core-default.xml | 24 + .../hadoop-common/src/site/markdown/Metrics.md | 1 + .../src/site/markdown/SingleCluster.md.vm | 3 +- .../hadoop/fs/contract/ContractTestUtils.java | 420 +++++++++++++++ .../org/apache/hadoop/net/ServerSocketUtil.java | 39 ++ .../hadoop/security/TestLdapGroupsMapping.java | 140 +++++ .../hadoop/security/token/TestDtUtilShell.java | 24 + .../delegation/web/TestWebDelegationToken.java | 114 +++- .../key/kms/server/KMSAuthenticationFilter.java | 2 +- .../hadoop/crypto/key/kms/server/TestKMS.java | 127 ++++- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 38 +- .../hadoop/fs/http/server/HttpFSServer.java | 218 ++++---- .../server/blockmanagement/BlockManager.java | 4 + .../PendingReconstructionBlocks.java | 16 +- .../server/namenode/EncryptionZoneManager.java | 35 +- .../server/namenode/FSDirEncryptionZoneOp.java | 2 +- .../hadoop/hdfs/server/namenode/FSEditLog.java | 11 +- .../hdfs/server/namenode/FSNamesystem.java | 5 + .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 42 +- .../src/main/webapps/hdfs/explorer.html | 25 +- .../src/main/webapps/hdfs/explorer.js | 53 +- .../org/apache/hadoop/hdfs/TestDFSShell.java | 4 +- .../hadoop/hdfs/TestDFSUpgradeFromImage.java | 16 +- .../TestPendingReconstruction.java | 20 +- .../src/test/resources/testHDFSConf.xml | 2 +- .../hadoop/mapred/LocalContainerLauncher.java | 28 + .../v2/app/job/impl/TaskAttemptImpl.java | 5 +- .../v2/app/launcher/ContainerLauncherEvent.java | 21 +- .../v2/app/launcher/ContainerLauncherImpl.java | 19 +- .../v2/app/rm/RMContainerRequestor.java | 4 +- .../v2/app/launcher/TestContainerLauncher.java | 10 +- .../app/launcher/TestContainerLauncherImpl.java | 8 + .../hadoop/mapred/ResourceMgrDelegate.java | 5 +- .../hadoop/mapred/TestClientRedirect.java | 2 +- .../apache/hadoop/mapreduce/v2/TestMRJobs.java | 119 +++++ .../org/apache/hadoop/fs/s3a/Constants.java | 11 +- .../fs/s3a/ProgressableProgressListener.java | 94 ++++ .../hadoop/fs/s3a/S3AFastOutputStream.java | 65 +-- .../org/apache/hadoop/fs/s3a/S3AFileStatus.java | 7 + .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 529 ++++++++++++++----- .../hadoop/fs/s3a/S3AInstrumentation.java | 218 +++++--- .../apache/hadoop/fs/s3a/S3AOutputStream.java | 98 +--- .../hadoop/fs/s3a/S3AStorageStatistics.java | 104 ++++ .../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 48 ++ .../org/apache/hadoop/fs/s3a/Statistic.java | 143 +++++ .../src/site/markdown/tools/hadoop-aws/index.md | 53 +- .../hadoop/fs/s3a/AbstractS3ATestBase.java | 103 ++++ .../apache/hadoop/fs/s3a/S3ATestConstants.java | 76 +++ .../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 226 +++++++- .../apache/hadoop/fs/s3a/TestS3AEncryption.java | 104 ++++ .../TestS3AEncryptionAlgorithmPropagation.java | 82 +++ .../s3a/TestS3AEncryptionFastOutputStream.java | 35 ++ .../hadoop/fs/s3a/TestS3AFileOperationCost.java | 191 +++++++ .../fs/s3a/TestS3AFileSystemContract.java | 1 - .../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 130 +---- .../fs/s3a/scale/TestS3ADeleteManyFiles.java | 10 +- .../s3a/scale/TestS3ADirectoryPerformance.java | 189 +++++++ .../scale/TestS3AInputStreamPerformance.java | 6 +- .../src/test/resources/log4j.properties | 4 +- .../yarn/api/ApplicationClientProtocol.java | 2 +- .../yarn/api/ContainerManagementProtocol.java | 5 + .../SignalContainerResponse.java | 2 +- .../yarn/api/records/ExecutionTypeRequest.java | 124 +++++ .../yarn/api/records/ResourceRequest.java | 34 +- .../main/proto/applicationclient_protocol.proto | 2 +- .../proto/containermanagement_protocol.proto | 1 + .../src/main/proto/yarn_protos.proto | 7 +- .../hadoop/yarn/client/api/YarnClient.java | 2 +- .../yarn/client/api/impl/YarnClientImpl.java | 4 +- .../hadoop/yarn/client/cli/ApplicationCLI.java | 6 +- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 17 +- .../api/impl/TestDistributedScheduling.java | 9 +- .../yarn/client/api/impl/TestYarnClient.java | 4 +- .../hadoop/yarn/client/cli/TestLogsCLI.java | 70 ++- .../yarn/api/ContainerManagementProtocolPB.java | 7 + .../ApplicationClientProtocolPBClientImpl.java | 4 +- ...ContainerManagementProtocolPBClientImpl.java | 19 + .../ApplicationClientProtocolPBServiceImpl.java | 5 +- ...ontainerManagementProtocolPBServiceImpl.java | 20 + .../impl/pb/ExecutionTypeRequestPBImpl.java | 93 ++++ .../yarn/api/records/impl/pb/ProtoUtils.java | 15 + .../records/impl/pb/ResourceRequestPBImpl.java | 52 +- .../logaggregation/AggregatedLogFormat.java | 89 +++- .../logaggregation/ContainerLogsRequest.java | 13 +- .../yarn/logaggregation/LogCLIHelpers.java | 25 +- .../hadoop/yarn/TestContainerLaunchRPC.java | 10 + .../yarn/TestContainerResourceIncreaseRPC.java | 8 + .../java/org/apache/hadoop/yarn/TestRPC.java | 10 + .../hadoop/yarn/api/TestPBImplRecords.java | 2 + .../webapp/AHSWebServices.java | 50 +- .../webapp/TestAHSWebServices.java | 66 +++ .../containermanager/ContainerManagerImpl.java | 38 +- .../monitor/ContainerMetrics.java | 18 +- .../monitor/ContainersMonitorImpl.java | 16 +- .../nodemanager/scheduler/LocalScheduler.java | 3 +- .../nodemanager/webapp/NMWebServices.java | 49 +- .../yarn/server/nodemanager/TestEventFlow.java | 3 + .../TestNodeStatusUpdaterForLabels.java | 7 + .../amrmproxy/MockResourceManagerFacade.java | 2 +- .../BaseContainerManagerTest.java | 3 + .../monitor/TestContainerMetrics.java | 4 +- .../scheduler/TestLocalScheduler.java | 11 +- .../nodemanager/webapp/TestNMWebServices.java | 48 +- .../server/resourcemanager/ClientRMService.java | 2 +- .../reservation/NoOverCommitPolicy.java | 38 +- .../RLESparseResourceAllocation.java | 6 +- .../planning/StageAllocatorGreedyRLE.java | 16 +- .../rmapp/attempt/RMAppAttemptImpl.java | 28 +- .../scheduler/AbstractYarnScheduler.java | 8 - .../security/DelegationTokenRenewer.java | 27 +- .../yarn/server/resourcemanager/MockRM.java | 20 +- .../server/resourcemanager/NodeManager.java | 9 +- .../resourcemanager/TestAMAuthorization.java | 8 + .../TestApplicationMasterLauncher.java | 8 + .../TestDistributedSchedulingService.java | 17 +- .../resourcemanager/TestSignalContainer.java | 2 +- .../TestWorkPreservingRMRestart.java | 92 ++++ .../TestRLESparseResourceAllocation.java | 22 + .../planning/TestSimpleCapacityReplanner.java | 2 +- .../attempt/TestRMAppAttemptTransitions.java | 3 +- .../security/TestDelegationTokenRenewer.java | 98 ++++ 135 files changed, 5155 insertions(+), 935 deletions(-) ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org