Merge remote-tracking branch 'apache/trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ebe8c7c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ebe8c7c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ebe8c7c Branch: refs/heads/HDFS-7240 Commit: 2ebe8c7cb2aee1d2779183c1364dc14ad0baa0df Parents: ef128ee 3e6fce9 Author: Arpit Agarwal <[email protected]> Authored: Mon Jul 27 14:57:03 2015 -0700 Committer: Arpit Agarwal <[email protected]> Committed: Mon Jul 27 14:57:03 2015 -0700 ---------------------------------------------------------------------- dev-support/releasedocmaker.py | 405 +++++---- hadoop-common-project/hadoop-common/CHANGES.txt | 16 + .../hadoop-common/src/JNIFlags.cmake | 124 --- .../apache/hadoop/fs/AbstractFileSystem.java | 13 + .../java/org/apache/hadoop/fs/FileContext.java | 20 + .../java/org/apache/hadoop/fs/FileSystem.java | 30 +- .../org/apache/hadoop/fs/FilterFileSystem.java | 6 + .../java/org/apache/hadoop/fs/FilterFs.java | 6 + .../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 6 + .../org/apache/hadoop/fs/viewfs/ViewFs.java | 15 + .../org/apache/hadoop/ipc/CallQueueManager.java | 27 +- .../hadoop-common/src/site/markdown/Metrics.md | 1 + .../src/site/markdown/filesystem/filesystem.md | 4 + .../hadoop/fs/FileSystemContractBaseTest.java | 11 +- .../org/apache/hadoop/fs/TestHarFileSystem.java | 3 + .../apache/hadoop/ipc/TestCallQueueManager.java | 6 +- .../org/apache/hadoop/net/ServerSocketUtil.java | 63 ++ .../org/apache/hadoop/hdfs/inotify/Event.java | 95 +++ .../hadoop/hdfs/protocol/ClientProtocol.java | 306 ++++--- .../hadoop/hdfs/web/ByteRangeInputStream.java | 57 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 19 + .../main/java/org/apache/hadoop/fs/Hdfs.java | 5 + .../java/org/apache/hadoop/hdfs/DFSClient.java | 18 + .../hadoop/hdfs/DistributedFileSystem.java | 19 + .../server/blockmanagement/BlockManager.java | 4 + .../blockmanagement/UnderReplicatedBlocks.java | 36 +- .../hdfs/server/namenode/FSNamesystem.java | 9 +- .../hadoop/hdfs/TestBlockStoragePolicy.java | 17 + .../hdfs/TestDFSInotifyEventInputStream.java | 30 +- .../hadoop/hdfs/TestDistributedFileSystem.java | 13 +- .../TestUnderReplicatedBlocks.java | 48 ++ .../hdfs/web/TestByteRangeInputStream.java | 35 +- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 41 + hadoop-yarn-project/CHANGES.txt | 44 + .../hadoop-yarn/hadoop-yarn-api/pom.xml | 34 + .../hadoop/yarn/conf/YarnConfiguration.java | 11 + .../yarn/conf/TestYarnConfigurationFields.java | 136 +++ .../hadoop-yarn/hadoop-yarn-common/pom.xml | 2 +- .../yarn/webapp/view/TwoColumnLayout.java | 2 +- .../main/proto/server/yarn_security_token.proto | 70 -- .../src/main/proto/yarn_security_token.proto | 70 ++ .../src/main/resources/yarn-default.xml | 16 + .../yarn/conf/TestYarnConfigurationFields.java | 136 --- .../pom.xml | 2 +- .../server/nodemanager/ContainerExecutor.java | 23 +- .../nodemanager/DefaultContainerExecutor.java | 2 +- .../nodemanager/DockerContainerExecutor.java | 2 +- .../nodemanager/LinuxContainerExecutor.java | 222 +++-- .../nodemanager/LocalDirsHandlerService.java | 35 +- .../launcher/ContainerLaunch.java | 15 + .../linux/privileged/PrivilegedOperation.java | 46 +- .../PrivilegedOperationException.java | 30 +- .../privileged/PrivilegedOperationExecutor.java | 30 +- .../linux/resources/CGroupsHandler.java | 8 + .../linux/resources/CGroupsHandlerImpl.java | 12 +- .../runtime/DefaultLinuxContainerRuntime.java | 148 ++++ .../DelegatingLinuxContainerRuntime.java | 110 +++ .../runtime/DockerLinuxContainerRuntime.java | 273 ++++++ .../linux/runtime/LinuxContainerRuntime.java | 38 + .../runtime/LinuxContainerRuntimeConstants.java | 69 ++ .../linux/runtime/docker/DockerClient.java | 82 ++ .../linux/runtime/docker/DockerCommand.java | 66 ++ .../linux/runtime/docker/DockerLoadCommand.java | 30 + .../linux/runtime/docker/DockerRunCommand.java | 107 +++ .../runtime/ContainerExecutionException.java | 85 ++ .../runtime/ContainerRuntime.java | 50 ++ .../runtime/ContainerRuntimeConstants.java | 33 + .../runtime/ContainerRuntimeContext.java | 105 +++ .../executor/ContainerLivenessContext.java | 13 + .../executor/ContainerReacquisitionContext.java | 13 + .../executor/ContainerSignalContext.java | 13 + .../executor/ContainerStartContext.java | 23 +- .../container-executor/impl/configuration.c | 17 +- .../container-executor/impl/configuration.h | 2 + .../impl/container-executor.c | 417 ++++++++-- .../impl/container-executor.h | 25 +- .../main/native/container-executor/impl/main.c | 97 ++- .../TestLinuxContainerExecutorWithMocks.java | 118 ++- .../TestPrivilegedOperationExecutor.java | 8 +- .../runtime/TestDockerContainerRuntime.java | 219 +++++ .../webapp/TestContainerLogsPage.java | 48 ++ .../hadoop-yarn-server-resourcemanager/pom.xml | 2 +- .../server/resourcemanager/RMContextImpl.java | 3 +- .../ProportionalCapacityPreemptionPolicy.java | 9 +- .../resourcemanager/recovery/RMStateStore.java | 9 +- .../reservation/AbstractReservationSystem.java | 2 + .../reservation/GreedyReservationAgent.java | 390 --------- .../reservation/InMemoryPlan.java | 13 +- .../InMemoryReservationAllocation.java | 8 +- .../resourcemanager/reservation/Plan.java | 1 + .../reservation/PlanContext.java | 2 + .../resourcemanager/reservation/PlanView.java | 31 +- .../resourcemanager/reservation/Planner.java | 47 -- .../RLESparseResourceAllocation.java | 55 +- .../reservation/ReservationAgent.java | 72 -- .../ReservationSchedulerConfiguration.java | 6 +- .../reservation/ReservationSystem.java | 5 +- .../reservation/ReservationSystemUtil.java | 6 +- .../reservation/SimpleCapacityReplanner.java | 113 --- .../planning/AlignedPlannerWithGreedy.java | 123 +++ .../planning/GreedyReservationAgent.java | 97 +++ .../reservation/planning/IterativePlanner.java | 338 ++++++++ .../reservation/planning/Planner.java | 49 ++ .../reservation/planning/PlanningAlgorithm.java | 207 +++++ .../reservation/planning/ReservationAgent.java | 73 ++ .../planning/SimpleCapacityReplanner.java | 118 +++ .../reservation/planning/StageAllocator.java | 55 ++ .../planning/StageAllocatorGreedy.java | 152 ++++ .../planning/StageAllocatorLowCostAligned.java | 360 ++++++++ .../planning/StageEarliestStart.java | 46 + .../planning/StageEarliestStartByDemand.java | 106 +++ .../StageEarliestStartByJobArrival.java | 39 + .../planning/TryManyReservationAgents.java | 114 +++ .../scheduler/ResourceLimits.java | 19 +- .../scheduler/capacity/AbstractCSQueue.java | 27 +- .../scheduler/capacity/CSAssignment.java | 12 +- .../capacity/CapacityHeadroomProvider.java | 16 +- .../scheduler/capacity/CapacityScheduler.java | 16 +- .../scheduler/capacity/LeafQueue.java | 833 +++---------------- .../scheduler/capacity/ParentQueue.java | 16 +- .../scheduler/capacity/ReservationQueue.java | 4 - .../scheduler/common/fica/FiCaSchedulerApp.java | 721 +++++++++++++++- .../webapp/CapacitySchedulerPage.java | 7 +- .../webapp/DefaultSchedulerPage.java | 4 +- .../webapp/FairSchedulerPage.java | 10 +- .../webapp/dao/FairSchedulerQueueInfo.java | 4 +- ...estProportionalCapacityPreemptionPolicy.java | 6 +- .../resourcemanager/recovery/TestProtos.java | 36 + .../reservation/ReservationSystemTestUtil.java | 5 +- .../reservation/TestCapacityOverTimePolicy.java | 2 +- .../TestCapacitySchedulerPlanFollower.java | 1 + .../reservation/TestFairReservationSystem.java | 1 - .../TestFairSchedulerPlanFollower.java | 1 + .../reservation/TestGreedyReservationAgent.java | 604 -------------- .../reservation/TestInMemoryPlan.java | 2 + .../reservation/TestNoOverCommitPolicy.java | 1 + .../TestRLESparseResourceAllocation.java | 51 +- .../TestSchedulerPlanFollowerBase.java | 1 + .../TestSimpleCapacityReplanner.java | 162 ---- .../planning/TestAlignedPlanner.java | 820 ++++++++++++++++++ .../planning/TestGreedyReservationAgent.java | 611 ++++++++++++++ .../planning/TestSimpleCapacityReplanner.java | 170 ++++ .../capacity/TestApplicationLimits.java | 15 +- .../capacity/TestCapacityScheduler.java | 3 +- .../capacity/TestContainerAllocation.java | 85 +- .../scheduler/capacity/TestLeafQueue.java | 191 +---- .../capacity/TestReservationQueue.java | 26 +- .../scheduler/capacity/TestReservations.java | 111 +-- .../scheduler/capacity/TestUtils.java | 25 +- .../webapp/dao/TestFairSchedulerQueueInfo.java | 59 ++ .../yarn/server/webproxy/AppReportFetcher.java | 79 +- .../server/webproxy/TestAppReportFetcher.java | 117 +++ 152 files changed, 8734 insertions(+), 3456 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ebe8c7c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ----------------------------------------------------------------------
