Merge branch 'trunk' into HDFS-1312
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0fd3f16b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0fd3f16b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0fd3f16b Branch: refs/heads/HDFS-1312 Commit: 0fd3f16be9c566aebf40a1ea0eea3ad99a638655 Parents: 5cc5406 1425578 Author: Arpit Agarwal <[email protected]> Authored: Wed Jan 20 10:48:32 2016 -0800 Committer: Arpit Agarwal <[email protected]> Committed: Wed Jan 20 10:48:32 2016 -0800 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 7 + .../src/main/conf/log4j.properties | 4 +- .../security/http/RestCsrfPreventionFilter.java | 89 ++ .../java/org/apache/hadoop/util/SysInfo.java | 11 +- .../org/apache/hadoop/util/SysInfoLinux.java | 15 +- .../org/apache/hadoop/util/SysInfoWindows.java | 26 +- .../src/site/markdown/filesystem/testing.md | 4 +- .../fs/contract/AbstractContractSeekTest.java | 8 +- .../hadoop/fs/contract/ContractOptions.java | 6 + .../http/TestRestCsrfPreventionFilter.java | 276 ++++++ .../apache/hadoop/util/TestSysInfoLinux.java | 18 +- .../apache/hadoop/util/TestSysInfoWindows.java | 51 +- .../security/http/RestCsrfPreventionFilter.java | 89 -- .../http/TestRestCsrfPreventionFilter.java | 276 ------ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 15 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 11 +- .../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 15 +- .../namenode/ha/TestFailureToReadEdits.java | 41 +- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 2 +- .../v2/app/TaskAttemptFinishingMonitor.java | 2 +- .../mapred/TestTaskAttemptFinishingMonitor.java | 2 +- .../mapred/TestTaskAttemptListenerImpl.java | 4 +- .../apache/hadoop/mapreduce/v2/app/MRApp.java | 10 +- .../hadoop/mapreduce/v2/app/TestRecovery.java | 2 +- .../mapreduce/v2/app/TestRuntimeEstimators.java | 2 +- .../v2/app/TestTaskHeartbeatHandler.java | 2 +- .../app/commit/TestCommitterEventHandler.java | 2 +- .../mapreduce/v2/app/job/impl/TestJobImpl.java | 8 +- .../v2/app/job/impl/TestShuffleProvider.java | 2 +- .../v2/app/job/impl/TestTaskAttempt.java | 28 +- .../impl/TestTaskAttemptContainerRequest.java | 2 +- .../mapreduce/v2/app/job/impl/TestTaskImpl.java | 2 +- .../v2/app/rm/TestRMContainerAllocator.java | 14 +- .../mapreduce/v2/hs/HistoryFileManager.java | 2 +- .../mapreduce/v2/hs/TestHistoryFileManager.java | 5 +- .../v2/TestSpeculativeExecutionWithMRApp.java | 4 +- .../org/apache/hadoop/fs/s3/S3FileSystem.java | 58 +- .../org/apache/hadoop/fs/s3/S3InputStream.java | 17 +- .../hadoop/fs/contract/s3/S3Contract.java | 40 + .../fs/contract/s3/TestS3ContractCreate.java | 32 + .../fs/contract/s3/TestS3ContractDelete.java | 31 + .../fs/contract/s3/TestS3ContractMkdir.java | 32 + .../fs/contract/s3/TestS3ContractOpen.java | 32 + .../fs/contract/s3/TestS3ContractRename.java | 32 + .../fs/contract/s3/TestS3ContractRootDir.java | 34 + .../fs/contract/s3/TestS3ContractSeek.java | 32 + .../src/test/resources/contract/s3.xml | 104 +++ .../fs/azure/AzureNativeFileSystemStore.java | 23 +- .../hadoop/fs/azure/BlockBlobAppendStream.java | 775 ++++++++++++++++ .../hadoop/fs/azure/NativeAzureFileSystem.java | 216 ++--- .../fs/azure/NativeAzureFileSystemHelper.java | 107 +++ .../hadoop/fs/azure/NativeFileSystemStore.java | 2 + .../hadoop/fs/azure/PageBlobOutputStream.java | 17 +- .../hadoop/fs/azure/StorageInterface.java | 89 +- .../hadoop/fs/azure/StorageInterfaceImpl.java | 33 +- .../hadoop-azure/src/site/markdown/index.md | 20 +- .../hadoop/fs/azure/MockStorageInterface.java | 34 +- .../azure/TestNativeAzureFileSystemAppend.java | 362 ++++++++ .../gridmix/DummyResourceCalculatorPlugin.java | 2 +- .../sls/scheduler/ResourceSchedulerWrapper.java | 4 +- hadoop-yarn-project/CHANGES.txt | 25 +- .../hadoop/yarn/conf/YarnConfiguration.java | 62 +- .../distributedshell/ApplicationMaster.java | 25 +- .../yarn/util/ProcfsBasedProcessTree.java | 4 +- .../yarn/util/ResourceCalculatorPlugin.java | 15 +- .../apache/hadoop/yarn/util/SystemClock.java | 12 +- .../yarn/util/WindowsBasedProcessTree.java | 2 +- .../src/main/resources/yarn-default.xml | 58 ++ .../hadoop/yarn/util/ControlledClock.java | 2 +- .../yarn/util/TestProcfsBasedProcessTree.java | 9 +- .../yarn/util/TestWindowsBasedProcessTree.java | 2 +- .../pom.xml | 11 + .../ApplicationHistoryServer.java | 5 +- .../server/timeline/TimelineDataManager.java | 1 - .../timeline/TimelineDataManagerMetrics.java | 11 +- .../TestApplicationHistoryClientService.java | 1 + ...pplicationHistoryManagerOnTimelineStore.java | 1 + .../webapp/TestAHSWebServices.java | 1 + .../timeline/TestTimelineDataManager.java | 1 + .../nodemanager/NodeResourceMonitorImpl.java | 4 +- .../container/ContainerImpl.java | 2 +- .../linux/resources/CGroupsHandlerImpl.java | 2 +- .../util/CgroupsLCEResourcesHandler.java | 2 +- .../monitor/MockResourceCalculatorPlugin.java | 2 +- .../util/TestNodeManagerHardwareUtils.java | 2 +- .../resourcemanager/NodesListManager.java | 3 +- .../resourcemanager/RMActiveServiceContext.java | 2 +- .../metrics/SystemMetricsPublisher.java | 14 +- .../ProportionalCapacityPreemptionPolicy.java | 12 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 2 +- .../rmcontainer/RMContainerImpl.java | 17 +- .../scheduler/AbstractYarnScheduler.java | 44 +- .../scheduler/AppSchedulingInfo.java | 17 +- .../scheduler/ContainerPreemptEvent.java | 57 -- .../scheduler/PreemptableResourceScheduler.java | 6 +- .../resourcemanager/scheduler/QueueMetrics.java | 56 +- .../scheduler/SchedulerApplicationAttempt.java | 4 +- .../scheduler/capacity/AbstractCSQueue.java | 24 + .../scheduler/capacity/CSQueue.java | 7 + .../scheduler/capacity/CapacityScheduler.java | 86 +- .../CapacitySchedulerConfiguration.java | 13 + .../scheduler/capacity/LeafQueue.java | 42 +- .../scheduler/capacity/UserInfo.java | 8 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 2 +- .../scheduler/event/ContainerPreemptEvent.java | 56 ++ .../event/ContainerRescheduledEvent.java | 35 - .../scheduler/event/SchedulerEventType.java | 11 +- .../fair/AllocationFileLoaderService.java | 2 +- .../scheduler/fair/FairScheduler.java | 36 +- .../scheduler/fifo/FifoScheduler.java | 28 +- .../webapp/CapacitySchedulerPage.java | 43 +- .../webapp/dao/CapacitySchedulerInfo.java | 2 +- .../dao/CapacitySchedulerLeafQueueInfo.java | 23 +- .../webapp/dao/CapacitySchedulerQueueInfo.java | 12 +- .../dao/PartitionQueueCapacitiesInfo.java | 12 +- .../webapp/dao/PartitionResourceUsageInfo.java | 89 -- .../webapp/dao/PartitionResourcesInfo.java | 99 ++ .../webapp/dao/QueueCapacitiesInfo.java | 17 +- .../webapp/dao/ResourceUsageInfo.java | 79 -- .../webapp/dao/ResourcesInfo.java | 81 ++ .../yarn/server/resourcemanager/MockRM.java | 3 +- .../resourcemanager/TestRMDispatcher.java | 20 +- .../TestWorkPreservingRMRestart.java | 2 +- .../applicationsmanager/TestAMRestart.java | 8 +- ...estProportionalCapacityPreemptionPolicy.java | 10 +- .../rmapp/TestNodesListManager.java | 2 +- .../rmapp/attempt/TestAMLivelinessMonitor.java | 2 +- .../scheduler/TestAbstractYarnScheduler.java | 113 +++ .../scheduler/TestQueueMetrics.java | 8 + .../capacity/TestApplicationLimits.java | 15 +- .../capacity/TestApplicationPriority.java | 6 +- .../capacity/TestCapacityScheduler.java | 8 +- .../scheduler/capacity/TestLeafQueue.java | 27 +- .../scheduler/fair/TestFSParentQueue.java | 2 +- .../scheduler/fair/TestFairScheduler.java | 16 +- .../scheduler/fair/TestQueueManager.java | 2 +- .../policy/TestFairOrderingPolicy.java | 54 ++ .../TestRMWebServicesForCSWithPartitions.java | 4 +- .../webapp/dao/TestFairSchedulerQueueInfo.java | 2 +- .../pom.xml | 136 +++ .../yarn/server/timeline/EntityCacheItem.java | 170 ++++ .../timeline/EntityGroupFSTimelineStore.java | 895 +++++++++++++++++++ .../hadoop/yarn/server/timeline/LogInfo.java | 281 ++++++ .../timeline/TimelineEntityGroupPlugin.java | 74 ++ .../yarn/server/timeline/package-info.java | 23 + .../timeline/EntityGroupPlugInForTest.java | 56 ++ .../server/timeline/PluginStoreTestUtils.java | 208 +++++ .../TestEntityGroupFSTimelineStore.java | 332 +++++++ .../yarn/server/timeline/TestLogInfo.java | 253 ++++++ .../hadoop-yarn/hadoop-yarn-server/pom.xml | 1 + 150 files changed, 6065 insertions(+), 1156 deletions(-) ----------------------------------------------------------------------
