Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/070ad843 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/070ad843 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/070ad843 Branch: refs/heads/HDFS-7240 Commit: 070ad8438b29ae1044bb5cccc3aebcdeba93f95d Parents: 4541ead 2abab1d Author: Xiaoyu Yao <[email protected]> Authored: Tue Dec 12 15:54:07 2017 -0800 Committer: Xiaoyu Yao <[email protected]> Committed: Tue Dec 12 15:54:07 2017 -0800 ---------------------------------------------------------------------- .../hadoop/fs/StreamCapabilitiesPolicy.java | 15 +- .../io/erasurecode/ErasureCodeConstants.java | 2 +- .../org/apache/hadoop/security/Credentials.java | 112 ++- .../hadoop/security/token/DtFileOperations.java | 10 +- .../org/apache/hadoop/util/StringUtils.java | 31 + .../fs/contract/AbstractContractOpenTest.java | 53 +- .../apache/hadoop/net/TestClusterTopology.java | 58 +- .../hadoop/security/token/TestDtUtilShell.java | 15 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 74 +- .../hadoop/hdfs/DistributedFileSystem.java | 28 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 15 + .../hadoop/hdfs/protocol/HdfsPathHandle.java | 65 +- .../ClientNamenodeProtocolTranslatorPB.java | 25 +- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 15 - .../src/main/proto/ClientNamenodeProtocol.proto | 11 + .../src/main/proto/hdfs.proto | 2 + ...tNamenodeProtocolServerSideTranslatorPB.java | 22 + .../blockmanagement/InvalidateBlocks.java | 8 +- .../hdfs/server/datanode/BlockReceiver.java | 5 + .../federation/router/RouterRpcServer.java | 47 ++ .../hdfs/server/namenode/FSDirAppendOp.java | 3 +- .../hdfs/server/namenode/FSDirSnapshotOp.java | 5 +- .../server/namenode/FSDirStatAndListingOp.java | 41 +- .../hdfs/server/namenode/FSDirWriteFileOp.java | 23 +- .../hadoop/hdfs/server/namenode/FSEditLog.java | 3 +- .../hdfs/server/namenode/FSEditLogLoader.java | 2 +- .../hdfs/server/namenode/FSEditLogOp.java | 26 + .../hdfs/server/namenode/FSNamesystem.java | 16 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 14 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 24 +- .../hdfs/server/namenode/NamenodeFsck.java | 2 +- .../hdfs/tools/DelegationTokenFetcher.java | 3 +- .../hdfs/tools/federation/RouterAdmin.java | 2 +- .../src/main/webapps/router/robots.txt | 2 + .../src/site/markdown/HDFSCommands.md | 2 +- .../src/site/markdown/HDFSRouterFederation.md | 4 + .../java/org/apache/hadoop/fs/TestUnbuffer.java | 20 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 23 + .../TestDistributedFileSystemWithECFile.java | 55 ++ .../fsdataset/impl/LazyPersistTestCase.java | 1 - .../server/federation/FederationTestUtils.java | 60 +- .../server/federation/RouterDFSCluster.java | 5 +- .../resolver/TestMountTableResolver.java | 34 +- .../federation/router/TestRouterAdmin.java | 31 + .../federation/router/TestRouterMountTable.java | 143 ++++ .../hdfs/server/namenode/NameNodeAdapter.java | 7 +- .../namenode/OfflineEditsViewerHelper.java | 4 +- .../hdfs/server/namenode/TestBackupNode.java | 8 +- .../hdfs/server/namenode/TestEditLog.java | 3 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 2 +- .../server/namenode/TestNamenodeRetryCache.java | 14 +- .../hdfs/server/namenode/TestReencryption.java | 5 +- .../hdfs/server/namenode/TestSaveNamespace.java | 2 +- .../server/namenode/ha/TestEditLogTailer.java | 4 +- .../namenode/ha/TestEditLogsDuringFailover.java | 15 +- .../namenode/ha/TestFailureToReadEdits.java | 12 +- .../namenode/ha/TestInitializeSharedEdits.java | 2 +- .../namenode/ha/TestRetryCacheWithHA.java | 13 +- .../namenode/ha/TestStandbyInProgressTail.java | 69 +- .../src/test/resources/contract/hdfs.xml | 2 +- .../hadoop-hdfs/src/test/resources/editsStored | Bin 6753 -> 7909 bytes .../src/test/resources/editsStored.xml | 536 +++++++++---- .../org/apache/hadoop/mapred/ClientCache.java | 6 +- .../hadoop/mapred/ClientServiceDelegate.java | 7 +- .../hadoop/mapred/ResourceMgrDelegate.java | 7 +- .../org/apache/hadoop/mapred/YARNRunner.java | 6 +- .../org/apache/hadoop/fi/ProbabilityModel.java | 7 +- .../apache/hadoop/fs/AccumulatingReducer.java | 8 +- .../java/org/apache/hadoop/fs/DFSCIOTest.java | 6 +- .../apache/hadoop/fs/DistributedFSCheck.java | 7 +- .../org/apache/hadoop/fs/JHLogAnalyzer.java | 7 +- .../java/org/apache/hadoop/fs/TestDFSIO.java | 6 +- .../java/org/apache/hadoop/fs/TestJHLA.java | 7 +- .../fs/loadGenerator/LoadGeneratorMR.java | 6 +- .../org/apache/hadoop/fs/slive/AppendOp.java | 6 +- .../apache/hadoop/fs/slive/ConfigExtractor.java | 7 +- .../org/apache/hadoop/fs/slive/CreateOp.java | 6 +- .../org/apache/hadoop/fs/slive/DeleteOp.java | 6 +- .../java/org/apache/hadoop/fs/slive/ListOp.java | 6 +- .../org/apache/hadoop/fs/slive/MkdirOp.java | 6 +- .../java/org/apache/hadoop/fs/slive/ReadOp.java | 6 +- .../org/apache/hadoop/fs/slive/RenameOp.java | 6 +- .../apache/hadoop/fs/slive/ReportWriter.java | 6 +- .../org/apache/hadoop/fs/slive/SleepOp.java | 6 +- .../org/apache/hadoop/fs/slive/SliveMapper.java | 6 +- .../apache/hadoop/fs/slive/SliveReducer.java | 6 +- .../org/apache/hadoop/fs/slive/SliveTest.java | 6 +- .../org/apache/hadoop/fs/slive/TestSlive.java | 10 +- .../org/apache/hadoop/fs/slive/TruncateOp.java | 6 +- .../apache/hadoop/fs/slive/WeightSelector.java | 7 +- .../java/org/apache/hadoop/hdfs/NNBench.java | 7 +- .../apache/hadoop/hdfs/NNBenchWithoutMR.java | 8 +- .../org/apache/hadoop/mapred/BigMapOutput.java | 8 +- .../java/org/apache/hadoop/mapred/MRBench.java | 6 +- .../org/apache/hadoop/mapred/MiniMRCluster.java | 15 +- .../hadoop/mapred/MiniMRYarnClusterAdapter.java | 7 +- .../apache/hadoop/mapred/ReliabilityTest.java | 13 +- .../apache/hadoop/mapred/TestBadRecords.java | 8 +- .../hadoop/mapred/TestClientRedirect.java | 7 +- .../mapred/TestCombineFileInputFormat.java | 11 +- .../TestCombineSequenceFileInputFormat.java | 8 +- .../mapred/TestCombineTextInputFormat.java | 8 +- .../mapred/TestConcatenatedCompressedInput.java | 9 +- .../mapred/TestFixedLengthInputFormat.java | 11 +- .../apache/hadoop/mapred/TestJobCleanup.java | 9 +- .../hadoop/mapred/TestJobSysDirWithDFS.java | 8 +- .../mapred/TestKeyValueTextInputFormat.java | 8 +- .../mapred/TestMRTimelineEventHandling.java | 8 +- .../apache/hadoop/mapred/TestMapProgress.java | 8 +- .../hadoop/mapred/TestMiniMRChildTask.java | 8 +- .../hadoop/mapred/TestMultiFileInputFormat.java | 8 +- .../TestSequenceFileAsBinaryOutputFormat.java | 13 +- .../apache/hadoop/mapred/TestSortedRanges.java | 12 +- .../TestSpecialCharactersInOutputPath.java | 15 +- .../apache/hadoop/mapred/TestTaskStatus.java | 4 +- .../hadoop/mapred/TestTextInputFormat.java | 11 +- .../apache/hadoop/mapred/TestYARNRunner.java | 14 +- .../hadoop/mapred/ThreadedMapBenchmark.java | 7 +- .../org/apache/hadoop/mapred/UtilsForTests.java | 6 +- .../mapred/jobcontrol/TestLocalJobControl.java | 9 +- .../apache/hadoop/mapred/pipes/TestPipes.java | 8 +- .../hadoop/mapreduce/GrowingSleepJob.java | 12 +- .../hadoop/mapreduce/JobHistoryFileParser.java | 7 +- .../mapreduce/JobHistoryFileReplayHelper.java | 9 +- .../mapreduce/JobHistoryFileReplayMapperV1.java | 9 +- .../mapreduce/JobHistoryFileReplayMapperV2.java | 8 +- .../hadoop/mapreduce/MapReduceTestUtil.java | 13 +- .../mapreduce/MiniHadoopClusterManager.java | 8 +- .../hadoop/mapreduce/SimpleEntityWriterV1.java | 7 +- .../hadoop/mapreduce/SimpleEntityWriterV2.java | 9 +- .../apache/hadoop/mapreduce/TestCounters.java | 11 +- .../hadoop/mapreduce/TestLocalRunner.java | 11 +- .../hadoop/mapreduce/TestMRJobClient.java | 7 +- .../hadoop/mapreduce/TestMapCollection.java | 13 +- .../hadoop/mapreduce/TestValueIterReset.java | 8 +- .../mapreduce/TimelineEntityConverterV1.java | 8 +- .../mapreduce/TimelineEntityConverterV2.java | 8 +- .../lib/db/TestDataDrivenDBInputFormat.java | 6 +- .../TestCombineSequenceFileInputFormat.java | 8 +- .../lib/input/TestCombineTextInputFormat.java | 8 +- .../lib/input/TestFixedLengthInputFormat.java | 12 +- .../input/TestMRKeyValueTextInputFormat.java | 12 +- .../input/TestMRSequenceFileInputFilter.java | 8 +- .../lib/jobcontrol/TestMapReduceJobControl.java | 12 +- .../TestMRSequenceFileAsBinaryOutputFormat.java | 8 +- .../lib/partition/TestKeyFieldHelper.java | 10 +- .../mapreduce/security/TestJHSSecurity.java | 10 +- .../mapreduce/util/MRAsyncDiskService.java | 7 +- .../mapreduce/util/TestMRAsyncDiskService.java | 9 +- .../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 7 +- .../TestMRAMWithNonNormalizedCapabilities.java | 7 +- .../mapreduce/v2/TestMRAppWithCombiner.java | 7 +- .../apache/hadoop/mapreduce/v2/TestMRJobs.java | 6 +- .../v2/TestMRJobsWithHistoryService.java | 8 +- .../mapreduce/v2/TestMRJobsWithProfiler.java | 8 +- .../hadoop/mapreduce/v2/TestMROldApiJobs.java | 7 +- .../hadoop/mapreduce/v2/TestRMNMInfo.java | 7 +- .../mapreduce/v2/TestSpeculativeExecution.java | 7 +- .../apache/hadoop/mapreduce/v2/TestUberAM.java | 6 +- .../test/java/testjar/UserNamePermission.java | 13 +- .../mapred/nativetask/HadoopPlatform.java | 7 +- .../mapred/nativetask/NativeBatchProcessor.java | 9 +- .../NativeMapOutputCollectorDelegator.java | 7 +- .../hadoop/mapred/nativetask/NativeRuntime.java | 7 +- .../hadoop/mapred/nativetask/Platforms.java | 6 +- .../mapred/nativetask/StatusReportChecker.java | 7 +- .../nativetask/handlers/BufferPuller.java | 6 +- .../nativetask/handlers/BufferPushee.java | 6 +- .../nativetask/handlers/BufferPusher.java | 6 +- .../nativetask/handlers/CombinerHandler.java | 7 +- .../handlers/NativeCollectorOnlyHandler.java | 7 +- .../mapred/nativetask/serde/KVSerializer.java | 7 +- .../mapred/nativetask/util/OutputUtil.java | 6 +- .../combinertest/LargeKVCombinerTest.java | 7 +- .../nativetask/combinertest/WordCount.java | 6 +- .../hadoop/mapred/nativetask/kvtest/KVJob.java | 6 +- .../hadoop/mapred/nativetask/kvtest/KVTest.java | 7 +- .../mapred/nativetask/kvtest/LargeKVTest.java | 6 +- .../mapred/nativetask/kvtest/TestInputFile.java | 7 +- .../EnforceNativeOutputCollectorDelegator.java | 7 +- .../mapred/uploader/FrameworkUploader.java | 59 +- .../mapred/uploader/TestFrameworkUploader.java | 79 +- hadoop-project/pom.xml | 2 +- .../apache/hadoop/fs/aliyun/oss/Constants.java | 2 +- .../apache/hadoop/tools/DistCpOptionSwitch.java | 2 +- .../src/site/markdown/DistCp.md.vm | 2 +- .../hadoop-yarn/conf/container-executor.cfg | 6 + .../yarn/api/records/QueueConfigurations.java | 76 ++ .../yarn/api/records/ResourceInformation.java | 6 + .../src/main/proto/yarn_protos.proto | 4 + .../hadoop/yarn/service/ServiceScheduler.java | 72 +- .../yarn/service/component/Component.java | 8 + .../component/instance/ComponentInstance.java | 10 +- .../yarn/service/conf/YarnServiceConf.java | 10 + .../hadoop/yarn/service/MockServiceAM.java | 144 +++- .../hadoop/yarn/service/TestServiceAM.java | 98 ++- .../yarn/client/api/async/AMRMClientAsync.java | 10 + .../api/async/impl/AMRMClientAsyncImpl.java | 5 + .../impl/pb/QueueConfigurationsPBImpl.java | 136 +++- .../api/records/impl/pb/ResourcePBImpl.java | 1 - .../resource/DefaultResourceCalculator.java | 14 + .../resource/DominantResourceCalculator.java | 46 ++ .../yarn/util/resource/ResourceCalculator.java | 32 +- .../hadoop/yarn/util/resource/Resources.java | 10 + .../types/yarn/YarnRegistryAttributes.java | 1 + .../hadoop/yarn/server/webapp/WebPageUtils.java | 4 +- .../yarn/server/MockResourceManagerFacade.java | 16 +- .../src/CMakeLists.txt | 2 + .../hadoop/yarn/server/nodemanager/Context.java | 3 + .../nodemanager/DefaultContainerExecutor.java | 25 +- .../yarn/server/nodemanager/NodeManager.java | 19 +- .../nodemanager/NodeResourceMonitorImpl.java | 16 +- .../nodemanager/amrmproxy/AMRMProxyService.java | 5 +- .../amrmproxy/FederationInterceptor.java | 271 ++++++- .../container/ContainerImpl.java | 20 +- .../launcher/ContainerLaunch.java | 54 +- .../JavaSandboxLinuxContainerRuntime.java | 17 +- .../monitor/ContainersMonitorImpl.java | 13 + .../nodemanager/metrics/NodeManagerMetrics.java | 61 ++ .../main/native/container-executor/impl/main.c | 6 + .../impl/modules/fpga/fpga-module.c | 229 ++++++ .../impl/modules/fpga/fpga-module.h | 45 ++ .../test/modules/fpga/test-fpga-module.cc | 216 ++++++ .../nodemanager/TestNodeResourceMonitor.java | 34 +- .../amrmproxy/BaseAMRMProxyTest.java | 21 + .../amrmproxy/TestFederationInterceptor.java | 104 +++ .../TestableFederationInterceptor.java | 8 +- .../monitor/MockResourceCalculatorPlugin.java | 5 + .../monitor/TestContainersMonitor.java | 20 +- .../server/resourcemanager/RMAppManager.java | 10 +- .../monitor/SchedulingMonitor.java | 4 +- .../AbstractPreemptableResourceCalculator.java | 24 +- .../capacity/FifoCandidatesSelector.java | 9 +- .../ProportionalCapacityPreemptionPolicy.java | 17 +- .../monitor/capacity/TempQueuePerPartition.java | 36 +- .../CapacitySchedulerPlanFollower.java | 12 +- .../scheduler/AbstractResourceUsage.java | 210 ++++++ .../scheduler/AbstractYarnScheduler.java | 2 +- .../scheduler/QueueResourceQuotas.java | 115 +++ .../scheduler/ResourceUsage.java | 243 ++---- .../scheduler/YarnScheduler.java | 2 +- .../capacity/AbstractAutoCreatedLeafQueue.java | 113 +++ .../scheduler/capacity/AbstractCSQueue.java | 265 ++++++- .../capacity/AbstractManagedParentQueue.java | 162 ++-- .../capacity/AutoCreatedLeafQueue.java | 147 ++-- .../capacity/AutoCreatedLeafQueueConfig.java | 66 ++ .../AutoCreatedQueueManagementPolicy.java | 64 ++ .../scheduler/capacity/CSQueue.java | 65 +- .../scheduler/capacity/CSQueueUtils.java | 65 +- .../scheduler/capacity/CapacityScheduler.java | 108 +-- .../CapacitySchedulerConfiguration.java | 358 ++++++++- .../capacity/CapacitySchedulerContext.java | 8 + .../capacity/CapacitySchedulerQueueManager.java | 6 +- .../scheduler/capacity/LeafQueue.java | 127 +++- .../scheduler/capacity/ManagedParentQueue.java | 294 +++++++- .../scheduler/capacity/ParentQueue.java | 305 +++++++- .../scheduler/capacity/PlanQueue.java | 150 +++- .../capacity/QueueManagementChange.java | 148 ++++ .../QueueManagementDynamicEditPolicy.java | 272 +++++++ .../scheduler/capacity/ReservationQueue.java | 91 +++ .../scheduler/capacity/UsersManager.java | 5 +- .../PriorityUtilizationQueueOrderingPolicy.java | 82 +- .../GuaranteedOrZeroCapacityOverTimePolicy.java | 745 +++++++++++++++++++ .../scheduler/common/QueueEntitlement.java | 22 + .../scheduler/common/fica/FiCaSchedulerApp.java | 28 +- .../event/QueueManagementChangeEvent.java | 49 ++ .../scheduler/event/SchedulerEventType.java | 5 +- .../webapp/CapacitySchedulerPage.java | 64 +- .../webapp/dao/CapacitySchedulerInfo.java | 3 +- .../dao/CapacitySchedulerLeafQueueInfo.java | 17 +- .../webapp/dao/CapacitySchedulerQueueInfo.java | 29 +- .../dao/PartitionQueueCapacitiesInfo.java | 34 +- .../webapp/dao/QueueCapacitiesInfo.java | 13 +- .../yarn/server/resourcemanager/MockNM.java | 8 + .../yarn/server/resourcemanager/MockRM.java | 6 + .../server/resourcemanager/TestAppManager.java | 112 ++- ...alCapacityPreemptionPolicyMockFramework.java | 27 + ...estProportionalCapacityPreemptionPolicy.java | 28 +- ...pacityPreemptionPolicyForNodePartitions.java | 76 ++ ...pacityPreemptionPolicyIntraQueueWithDRF.java | 6 +- .../TestAbsoluteResourceConfiguration.java | 630 ++++++++++++++++ .../capacity/TestApplicationLimits.java | 30 +- .../TestApplicationLimitsByPartition.java | 4 + .../capacity/TestAutoCreatedLeafQueue.java | 113 --- ...stCapacitySchedulerAutoCreatedQueueBase.java | 683 +++++++++++++++++ .../TestCapacitySchedulerAutoQueueCreation.java | 703 ++++++++--------- .../TestCapacitySchedulerDynamicBehavior.java | 32 +- ...CapacitySchedulerWithMultiResourceTypes.java | 25 +- .../scheduler/capacity/TestChildQueueOrder.java | 2 + ...tGuaranteedOrZeroCapacityOverTimePolicy.java | 40 + .../scheduler/capacity/TestLeafQueue.java | 492 +++++++----- .../TestNodeLabelContainerAllocation.java | 163 ++++ .../scheduler/capacity/TestParentQueue.java | 102 +++ .../TestQueueManagementDynamicEditPolicy.java | 121 +++ .../scheduler/capacity/TestQueueParsing.java | 168 ++++- .../capacity/TestReservationQueue.java | 114 +++ .../scheduler/capacity/TestReservations.java | 18 + .../scheduler/capacity/TestUtils.java | 2 +- ...tPriorityUtilizationQueueOrderingPolicy.java | 81 +- .../webapp/TestRMWebServicesCapacitySched.java | 4 +- .../src/site/markdown/CapacityScheduler.md | 10 +- .../markdown/yarn-service/Configurations.md | 1 + 302 files changed, 10651 insertions(+), 2587 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/070ad843/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/070ad843/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/070ad843/hadoop-project/pom.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
