Merge branch 'trunk' into HDFS-1312
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/20d8cf7c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/20d8cf7c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/20d8cf7c Branch: refs/heads/HDFS-1312 Commit: 20d8cf7c6ecb37332bf5cca696834b090a56e1cd Parents: 553daf1 5870611 Author: Anu Engineer <aengin...@apache.org> Authored: Wed Jun 1 12:54:42 2016 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Wed Jun 1 12:54:42 2016 -0700 ---------------------------------------------------------------------- dev-support/bin/qbt | 18 + dev-support/bin/yetus-wrapper | 2 +- hadoop-common-project/hadoop-auth/pom.xml | 28 +- .../authentication/util/KerberosUtil.java | 18 +- .../TestKerberosAuthenticationHandler.java | 1 - .../authentication/util/TestKerberosUtil.java | 26 +- .../dev-support/findbugsExcludeFile.xml | 8 +- hadoop-common-project/hadoop-common/pom.xml | 5 + .../key/kms/LoadBalancingKMSClientProvider.java | 20 +- .../hadoop/io/retry/AsyncCallHandler.java | 321 +++++++++++ .../org/apache/hadoop/io/retry/CallReturn.java | 75 +++ .../hadoop/io/retry/RetryInvocationHandler.java | 134 ++++- .../apache/hadoop/io/retry/RetryPolicies.java | 4 +- .../main/java/org/apache/hadoop/ipc/Client.java | 25 +- .../apache/hadoop/ipc/DecayRpcScheduler.java | 130 +++-- .../apache/hadoop/ipc/ProtobufRpcEngine.java | 18 +- .../main/java/org/apache/hadoop/ipc/RPC.java | 21 +- .../main/java/org/apache/hadoop/ipc/Server.java | 4 +- .../apache/hadoop/ipc/WritableRpcEngine.java | 564 +++++++++++++++++++ .../java/org/apache/hadoop/log/LogLevel.java | 285 +++++++++- .../java/org/apache/hadoop/security/KDiag.java | 34 +- .../hadoop/security/UserGroupInformation.java | 4 +- .../org/apache/hadoop/tracing/TraceAdmin.java | 16 +- .../java/org/apache/hadoop/util/ProtoUtil.java | 2 + .../apache/hadoop/util/concurrent/AsyncGet.java | 17 +- .../src/main/proto/RpcHeader.proto | 2 +- .../src/site/markdown/CommandsManual.md | 32 +- .../hadoop-common/src/site/markdown/Tracing.md | 9 + .../kms/TestLoadBalancingKMSClientProvider.java | 94 ++++ .../fs/FileContextCreateMkdirBaseTest.java | 73 ++- .../org/apache/hadoop/ipc/RPCCallBenchmark.java | 38 +- .../org/apache/hadoop/ipc/TestAsyncIPC.java | 10 +- .../hadoop/ipc/TestMultipleProtocolServer.java | 236 +++++++- .../java/org/apache/hadoop/ipc/TestRPC.java | 31 +- .../apache/hadoop/ipc/TestRPCCallBenchmark.java | 13 + .../apache/hadoop/ipc/TestRPCCompatibility.java | 242 +++++++- .../apache/hadoop/ipc/TestRPCWaitForProxy.java | 37 +- .../java/org/apache/hadoop/ipc/TestRpcBase.java | 50 +- .../java/org/apache/hadoop/ipc/TestSaslRPC.java | 91 +-- .../org/apache/hadoop/log/TestLogLevel.java | 479 +++++++++++++--- .../sink/RollingFileSystemSinkTestBase.java | 8 +- .../hadoop/security/TestDoAsEffectiveUser.java | 291 ++++++---- .../security/TestUserGroupInformation.java | 28 +- .../hadoop-common/src/test/proto/test.proto | 4 +- .../src/test/proto/test_rpc_service.proto | 4 +- .../hadoop-common/src/test/resources/krb5.conf | 23 +- .../hadoop/crypto/key/kms/server/TestKMS.java | 7 +- hadoop-common-project/hadoop-minikdc/pom.xml | 107 +--- .../java/org/apache/hadoop/minikdc/MiniKdc.java | 381 +++---------- .../src/main/resources/minikdc-krb5.conf | 25 - .../src/main/resources/minikdc.ldiff | 47 -- .../org/apache/hadoop/minikdc/TestMiniKdc.java | 19 +- .../hadoop/hdfs/AsyncDistributedFileSystem.java | 7 +- .../apache/hadoop/hdfs/FileChecksumHelper.java | 3 +- .../hadoop/hdfs/protocol/StripedBlockInfo.java | 10 +- .../hdfs/protocol/datatransfer/Sender.java | 2 + .../ClientNamenodeProtocolTranslatorPB.java | 42 +- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 16 + .../src/main/proto/datatransfer.proto | 1 + .../hdfs/protocol/datatransfer/Receiver.java | 1 + .../server/blockmanagement/BlockManager.java | 68 +-- .../blockmanagement/DecommissionManager.java | 8 +- .../server/datanode/BlockChecksumHelper.java | 172 ++++-- .../server/datanode/ShortCircuitRegistry.java | 6 +- .../erasurecode/ErasureCodingWorker.java | 15 +- .../StripedBlockChecksumReconstructor.java | 129 +++++ .../erasurecode/StripedBlockReconstructor.java | 119 ++++ .../datanode/erasurecode/StripedReader.java | 22 +- .../erasurecode/StripedReconstructionInfo.java | 99 ++++ .../erasurecode/StripedReconstructor.java | 169 ++---- .../datanode/erasurecode/StripedWriter.java | 29 +- .../hdfs/server/namenode/FSNamesystem.java | 2 +- .../server/namenode/FSPermissionChecker.java | 24 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 3 + .../hdfs/server/namenode/NamenodeFsck.java | 2 +- .../org/apache/hadoop/hdfs/TestAsyncDFS.java | 238 ++++++-- .../apache/hadoop/hdfs/TestAsyncDFSRename.java | 563 ++++-------------- .../apache/hadoop/hdfs/TestAsyncHDFSWithHA.java | 181 ++++++ .../apache/hadoop/hdfs/TestDFSPermission.java | 13 + .../hdfs/TestDecommissionWithStriped.java | 47 ++ .../hadoop/hdfs/TestDistributedFileSystem.java | 1 - .../apache/hadoop/hdfs/TestFileChecksum.java | 41 +- .../TestClientProtocolWithDelegationToken.java | 119 ++++ .../hdfs/server/namenode/ha/HATestUtil.java | 9 +- .../shortcircuit/TestShortCircuitCache.java | 34 +- .../sink/TestRollingFileSystemSinkWithHdfs.java | 93 +-- .../apache/hadoop/tracing/TestTraceAdmin.java | 69 ++- .../hadoop-hdfs/src/test/resources/krb5.conf | 19 +- .../jobhistory/JobHistoryEventHandler.java | 4 +- .../v2/app/job/impl/TaskAttemptImpl.java | 2 +- .../v2/app/rm/RMContainerAllocator.java | 16 +- .../v2/app/rm/ResourceCalculatorUtils.java | 17 +- .../CheckpointAMPreemptionPolicy.java | 8 +- .../v2/app/TestCheckpointPreemptionPolicy.java | 16 +- .../hadoop/mapreduce/v2/app/TestRecovery.java | 2 +- .../v2/app/job/impl/TestTaskAttempt.java | 29 +- .../app/local/TestLocalContainerAllocator.java | 2 +- .../v2/app/rm/TestRMContainerAllocator.java | 2 +- .../v2/app/rm/TestResourceCalculatorUtils.java | 4 +- .../apache/hadoop/mapreduce/TypeConverter.java | 6 +- .../org/apache/hadoop/mapreduce/JobStatus.java | 18 +- .../jobhistory/NormalizedResourceEvent.java | 6 +- .../org/apache/hadoop/mapreduce/tools/CLI.java | 6 +- .../mapreduce/v2/hs/server/HSAdminServer.java | 3 + .../apache/hadoop/mapred/JobClientUnitTest.java | 6 +- hadoop-project/pom.xml | 6 - .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 3 + .../src/site/markdown/DistCp.md.vm | 1 + .../sls/scheduler/FairSchedulerMetrics.java | 60 +- .../sls/scheduler/ResourceSchedulerWrapper.java | 36 +- .../sls/scheduler/SLSCapacityScheduler.java | 36 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 2 + .../hadoop/yarn/api/records/Container.java | 26 + .../hadoop/yarn/api/records/Resource.java | 34 +- .../hadoop/yarn/conf/YarnConfiguration.java | 24 + .../src/main/proto/yarn_protos.proto | 5 +- .../distributedshell/ApplicationMaster.java | 6 +- .../applications/distributedshell/Client.java | 4 +- .../hadoop/yarn/client/api/YarnClient.java | 4 +- .../yarn/client/api/impl/AMRMClientImpl.java | 16 +- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 357 ++++++------ .../apache/hadoop/yarn/client/cli/NodeCLI.java | 4 +- .../apache/hadoop/yarn/client/cli/TopCLI.java | 4 +- .../yarn/client/api/impl/TestYarnClient.java | 2 +- .../hadoop/yarn/client/cli/TestLogsCLI.java | 109 +++- .../api/records/impl/pb/ContainerPBImpl.java | 26 +- .../api/records/impl/pb/ResourcePBImpl.java | 30 +- .../logaggregation/AggregatedLogFormat.java | 14 +- .../logaggregation/ContainerLogsRequest.java | 124 ++++ .../yarn/logaggregation/LogCLIHelpers.java | 139 ++++- .../yarn/security/SchedulerSecurityInfo.java | 2 +- .../apache/hadoop/yarn/util/AdHocLogDumper.java | 9 +- .../resource/DefaultResourceCalculator.java | 36 +- .../resource/DominantResourceCalculator.java | 46 +- .../yarn/util/resource/ResourceCalculator.java | 10 +- .../hadoop/yarn/util/resource/Resources.java | 62 +- .../src/main/resources/yarn-default.xml | 16 + .../yarn/util/resource/TestResources.java | 8 +- .../api/DistributedSchedulerProtocolPB.java | 10 +- .../pb/DistSchedRegisterResponsePBImpl.java | 6 +- .../hadoop/yarn/server/utils/BuilderUtils.java | 12 +- .../hadoop/yarn/server/webapp/dao/AppInfo.java | 10 +- .../yarn/server/webapp/dao/ContainerInfo.java | 10 +- .../proto/distributed_scheduler_protocol.proto | 1 + .../hadoop/yarn/TestYarnServerApiClasses.java | 6 +- .../server/nodemanager/ContainerExecutor.java | 2 +- .../container/ContainerImpl.java | 4 +- .../CGroupsMemoryResourceHandlerImpl.java | 4 +- .../runtime/DockerLinuxContainerRuntime.java | 46 +- .../monitor/ContainersMonitorImpl.java | 16 +- .../queuing/QueuingContainerManagerImpl.java | 15 +- .../nodemanager/metrics/NodeManagerMetrics.java | 12 +- .../OpportunisticContainerAllocator.java | 3 +- .../nodemanager/webapp/dao/ContainerInfo.java | 2 +- .../nodemanager/TestNodeStatusUpdater.java | 4 +- .../runtime/TestDockerContainerRuntime.java | 298 ++++++++-- .../monitor/TestContainersMonitor.java | 2 +- .../queuing/TestQueuingContainerManager.java | 68 +-- .../yarn/server/resourcemanager/RMNMInfo.java | 4 +- .../server/resourcemanager/RMServerUtils.java | 14 +- .../resourcemanager/ResourceTrackerService.java | 2 +- .../metrics/SystemMetricsPublisher.java | 2 +- .../monitor/capacity/TempQueuePerPartition.java | 12 +- .../AbstractSchedulerPlanFollower.java | 2 +- .../reservation/CapacityOverTimePolicy.java | 6 +- .../reservation/ReservationSystemUtil.java | 2 +- .../planning/StageEarliestStartByDemand.java | 2 +- .../rmcontainer/RMContainerImpl.java | 2 +- .../scheduler/ClusterNodeTracker.java | 10 +- .../resourcemanager/scheduler/QueueMetrics.java | 61 +- .../scheduler/SchedulerApplicationAttempt.java | 6 +- .../scheduler/SchedulerUtils.java | 8 +- .../scheduler/capacity/CSQueueMetrics.java | 24 +- .../capacity/CapacityHeadroomProvider.java | 2 +- .../CapacitySchedulerConfiguration.java | 6 +- .../scheduler/capacity/LeafQueue.java | 2 +- .../allocator/RegularContainerAllocator.java | 2 +- .../scheduler/fair/FSAppAttempt.java | 2 +- .../scheduler/fair/FSParentQueue.java | 2 +- .../resourcemanager/scheduler/fair/FSQueue.java | 12 +- .../scheduler/fair/FSQueueMetrics.java | 41 +- .../scheduler/fair/FairScheduler.java | 39 +- .../fair/FairSchedulerConfiguration.java | 12 + .../fair/policies/ComputeFairShares.java | 18 +- .../DominantResourceFairnessPolicy.java | 10 +- .../fair/policies/FairSharePolicy.java | 20 +- .../scheduler/fair/policies/FifoPolicy.java | 8 +- .../scheduler/fifo/FifoScheduler.java | 15 +- .../scheduler/policy/FairOrderingPolicy.java | 5 +- .../webapp/DefaultSchedulerPage.java | 4 +- .../webapp/FairSchedulerAppsBlock.java | 2 +- .../resourcemanager/webapp/RMWebServices.java | 8 +- .../resourcemanager/webapp/dao/AppInfo.java | 20 +- .../webapp/dao/FairSchedulerInfo.java | 4 +- .../webapp/dao/FairSchedulerQueueInfo.java | 14 +- .../webapp/dao/FifoSchedulerInfo.java | 18 +- .../resourcemanager/webapp/dao/NodeInfo.java | 4 +- .../webapp/dao/ResourceInfo.java | 10 +- .../server/resourcemanager/Application.java | 2 +- .../yarn/server/resourcemanager/MockNM.java | 8 +- .../yarn/server/resourcemanager/MockNodes.java | 4 +- .../server/resourcemanager/NodeManager.java | 8 +- .../ReservationACLsTestBase.java | 2 +- .../resourcemanager/TestApplicationACLs.java | 6 +- .../resourcemanager/TestApplicationCleanup.java | 2 +- .../TestContainerResourceUsage.java | 2 +- .../TestDistributedSchedulingService.java | 149 +++-- .../yarn/server/resourcemanager/TestRM.java | 8 +- .../yarn/server/resourcemanager/TestRMHA.java | 4 +- .../resourcemanager/TestRMNodeTransitions.java | 22 +- .../TestReservationSystemWithRMHA.java | 2 +- .../TestResourceTrackerService.java | 2 +- .../TestWorkPreservingRMRestart.java | 29 +- .../metrics/TestSystemMetricsPublisher.java | 10 +- ...alCapacityPreemptionPolicyMockFramework.java | 4 +- ...estProportionalCapacityPreemptionPolicy.java | 39 +- ...alCapacityPreemptionPolicyMockFramework.java | 6 +- .../nodelabels/TestRMNodeLabelsManager.java | 2 +- .../reservation/ReservationSystemTestUtil.java | 2 +- .../TestRLESparseResourceAllocation.java | 18 +- .../planning/TestSimpleCapacityReplanner.java | 4 +- .../scheduler/TestAbstractYarnScheduler.java | 16 +- .../scheduler/TestQueueMetrics.java | 8 +- .../scheduler/TestResourceUsage.java | 2 +- .../scheduler/TestSchedulerUtils.java | 20 +- .../capacity/TestApplicationLimits.java | 6 +- .../capacity/TestApplicationPriority.java | 52 +- .../capacity/TestCapacityScheduler.java | 121 ++-- .../TestCapacitySchedulerLazyPreemption.java | 30 +- .../TestCapacitySchedulerNodeLabelUpdate.java | 52 +- ...TestCapacitySchedulerSurgicalPreemption.java | 4 +- .../scheduler/capacity/TestChildQueueOrder.java | 4 +- .../capacity/TestContainerAllocation.java | 53 +- .../capacity/TestContainerResizing.java | 150 ++--- .../capacity/TestIncreaseAllocationExpirer.java | 36 +- .../scheduler/capacity/TestLeafQueue.java | 492 ++++++++-------- .../TestNodeLabelContainerAllocation.java | 15 +- .../scheduler/capacity/TestParentQueue.java | 4 +- .../scheduler/capacity/TestQueueCapacities.java | 2 +- .../scheduler/capacity/TestReservations.java | 406 ++++++------- ...TestWorkPreservingRMRestartForNodeLabel.java | 4 +- .../scheduler/fair/FairSchedulerTestBase.java | 4 +- .../scheduler/fair/TestComputeFairShares.java | 2 +- .../fair/TestContinuousScheduling.java | 2 +- .../scheduler/fair/TestFSAppAttempt.java | 32 +- .../scheduler/fair/TestFSLeafQueue.java | 15 +- .../scheduler/fair/TestFairScheduler.java | 341 ++++++----- .../fair/TestFairSchedulerFairShare.java | 32 +- .../fair/TestFairSchedulerPreemption.java | 62 +- .../scheduler/fifo/TestFifoScheduler.java | 63 ++- .../webapp/TestRMWebServices.java | 17 +- .../webapp/TestRMWebServicesApps.java | 2 +- .../webapp/TestRMWebServicesNodes.java | 4 +- .../src/test/resources/krb5.conf | 23 +- .../yarn/server/timeline/EntityCacheItem.java | 66 ++- .../timeline/EntityGroupFSTimelineStore.java | 82 ++- .../timeline/EntityGroupPlugInForTest.java | 19 +- .../TestEntityGroupFSTimelineStore.java | 248 ++++++-- .../src/site/markdown/FairScheduler.md | 7 +- hadoop-yarn-project/hadoop-yarn/pom.xml | 7 + 260 files changed, 7734 insertions(+), 4019 deletions(-) ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org