Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d11161bb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d11161bb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d11161bb Branch: refs/heads/HDFS-7240 Commit: d11161bb6812c57905f21c96c89fb4e5d0ca2ca3 Parents: d6a4eae 792388e Author: Anu Engineer <[email protected]> Authored: Fri Oct 27 11:11:45 2017 -0700 Committer: Anu Engineer <[email protected]> Committed: Fri Oct 27 11:11:45 2017 -0700 ---------------------------------------------------------------------- LICENSE.txt | 1 + dev-support/docker/Dockerfile | 7 +- .../assemblies/hadoop-resourceestimator.xml | 42 + .../main/resources/assemblies/hadoop-tools.xml | 11 + .../jdiff/Apache_Hadoop_Common_2.8.2.xml | 38218 +++++++++++++++++ .../apache/hadoop/conf/ReconfigurableBase.java | 4 +- .../hadoop/conf/ReconfigurationTaskStatus.java | 6 +- .../java/org/apache/hadoop/net/NetUtils.java | 33 +- .../markdown/release/2.8.2/CHANGES.2.8.2.md | 55 +- .../release/2.8.2/RELEASENOTES.2.8.2.md | 5 +- .../apache/hadoop/conf/TestReconfiguration.java | 2 +- .../org/apache/hadoop/security/TestKDiag.java | 2 +- .../ReconfigurationProtocolUtils.java | 4 +- .../hadoop/hdfs/web/URLConnectionFactory.java | 61 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 16 +- .../jdiff/Apache_Hadoop_HDFS_2.8.2.xml | 312 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 7 + .../ReconfigurationProtocolServerSideUtils.java | 3 +- .../hdfs/qjournal/server/JournalNodeSyncer.java | 19 +- .../hdfs/server/namenode/FSNamesystem.java | 5 - .../hdfs/server/namenode/INodeDirectory.java | 18 + .../snapshot/DirectorySnapshottableFeature.java | 38 +- .../namenode/snapshot/SnapshotDiffInfo.java | 14 +- .../namenode/snapshot/SnapshotManager.java | 63 +- .../server/namenode/top/metrics/TopMetrics.java | 30 +- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 +- .../src/main/resources/hdfs-default.xml | 11 + .../server/namenode/metrics/TestTopMetrics.java | 11 +- .../snapshot/TestSnapRootDescendantDiff.java | 80 + .../snapshot/TestSnapshotDiffReport.java | 541 +- .../Apache_Hadoop_MapReduce_Common_2.8.2.xml | 113 + .../Apache_Hadoop_MapReduce_Core_2.8.2.xml | 27495 ++++++++++++ .../Apache_Hadoop_MapReduce_JobClient_2.8.2.xml | 16 + .../mapred/LocalDistributedCacheManager.java | 10 +- .../apache/hadoop/mapred/LocalJobRunner.java | 16 +- .../v2/jobhistory/FileNameIndexUtils.java | 9 +- .../v2/jobhistory/JobHistoryUtils.java | 7 +- .../v2/security/MRDelegationTokenRenewer.java | 8 +- .../security/client/ClientHSTokenSelector.java | 8 +- .../apache/hadoop/mapreduce/v2/util/MRApps.java | 6 +- .../hadoop/mapred/TestLocalModeWithNewApis.java | 8 +- .../mapred/TestMRWithDistributedCache.java | 8 +- hadoop-project-dist/pom.xml | 2 +- hadoop-project/pom.xml | 5 + hadoop-project/src/site/site.xml | 2 + hadoop-tools/hadoop-resourceestimator/README.md | 19 + hadoop-tools/hadoop-resourceestimator/pom.xml | 174 + .../src/config/checkstyle.xml | 50 + .../src/main/bin/estimator.cmd | 52 + .../src/main/bin/estimator.sh | 71 + .../src/main/bin/start-estimator.cmd | 37 + .../src/main/bin/start-estimator.sh | 42 + .../src/main/bin/stop-estimator.cmd | 37 + .../src/main/bin/stop-estimator.sh | 42 + .../src/main/conf/resourceestimator-config.xml | 85 + .../src/main/data/resourceEstimatorService.txt | 2 + .../common/api/RecurrenceId.java | 95 + .../common/api/ResourceSkyline.java | 211 + .../common/api/package-info.java | 23 + .../config/ResourceEstimatorConfiguration.java | 125 + .../common/config/ResourceEstimatorUtil.java | 81 + .../common/config/package-info.java | 23 + .../exception/ResourceEstimatorException.java | 35 + .../common/exception/package-info.java | 23 + .../RLESparseResourceAllocationSerDe.java | 77 + .../common/serialization/ResourceSerDe.java | 61 + .../common/serialization/package-info.java | 24 + .../service/ResourceEstimatorServer.java | 146 + .../service/ResourceEstimatorService.java | 238 + .../resourceestimator/service/ShutdownHook.java | 45 + .../resourceestimator/service/package-info.java | 23 + .../skylinestore/api/HistorySkylineStore.java | 99 + .../api/PredictionSkylineStore.java | 60 + .../skylinestore/api/SkylineStore.java | 30 + .../skylinestore/api/package-info.java | 23 + .../DuplicateRecurrenceIdException.java | 33 + .../EmptyResourceSkylineException.java | 33 + .../exceptions/NullPipelineIdException.java | 32 + ...ullRLESparseResourceAllocationException.java | 33 + .../exceptions/NullRecurrenceIdException.java | 32 + .../NullResourceSkylineException.java | 32 + .../RecurrenceIdNotFoundException.java | 33 + .../exceptions/SkylineStoreException.java | 33 + .../skylinestore/exceptions/package-info.java | 24 + .../skylinestore/impl/InMemoryStore.java | 256 + .../skylinestore/impl/package-info.java | 23 + .../validator/SkylineStoreValidator.java | 118 + .../skylinestore/validator/package-info.java | 23 + .../resourceestimator/solver/api/Solver.java | 76 + .../solver/api/package-info.java | 23 + .../exceptions/InvalidInputException.java | 34 + .../exceptions/InvalidSolverException.java | 34 + .../solver/exceptions/SolverException.java | 34 + .../solver/exceptions/package-info.java | 24 + .../solver/impl/BaseSolver.java | 94 + .../resourceestimator/solver/impl/LpSolver.java | 340 + .../solver/impl/package-info.java | 23 + .../solver/preprocess/SolverPreprocessor.java | 219 + .../solver/preprocess/package-info.java | 23 + .../translator/api/JobMetaData.java | 163 + .../translator/api/LogParser.java | 65 + .../translator/api/SingleLineParser.java | 52 + .../translator/api/package-info.java | 23 + .../exceptions/DataFieldNotFoundException.java | 32 + .../translator/exceptions/package-info.java | 23 + .../translator/impl/BaseLogParser.java | 125 + .../translator/impl/LogParserUtil.java | 97 + .../translator/impl/NativeSingleLineParser.java | 120 + .../translator/impl/RmSingleLineParser.java | 203 + .../translator/impl/package-info.java | 23 + .../translator/validator/ParserValidator.java | 41 + .../translator/validator/package-info.java | 23 + .../webapps/ResourceEstimatorServer/.gitignore | 14 + .../src/site/markdown/ResourceEstimator.md | 181 + .../src/site/resources/css/site.css | 29 + .../common/api/TestResourceSkyline.java | 128 + .../serialization/TestHistorySkylineSerDe.java | 134 + .../common/serialization/TestResourceSerDe.java | 64 + .../serialization/TestResourceSkylineSerDe.java | 112 + .../common/serialization/package-info.java | 24 + .../service/GuiceServletConfig.java | 42 + .../service/TestResourceEstimatorService.java | 282 + .../skylinestore/impl/TestInMemoryStore.java | 32 + .../skylinestore/impl/TestSkylineStore.java | 464 + .../solver/impl/TestLpSolver.java | 112 + .../solver/impl/TestSolver.java | 73 + .../translator/api/TestJobMetaData.java | 163 + .../translator/impl/TestNativeParser.java | 115 + .../translator/impl/TestRmParser.java | 239 + .../src/test/resources/log4j.properties | 25 + .../src/test/resources/lp/answer.txt | 7 + .../src/test/resources/lp/tinySample.txt | 2 + .../src/test/resources/lp/tpch_q12.txt | 13 + .../test/resources/resourceEstimatorService.txt | 2 + .../test/resources/resourceestimator-config.xml | 85 + .../src/test/resources/trace/invalidLog1.txt | 4 + .../src/test/resources/trace/invalidLog2.txt | 3 + .../src/test/resources/trace/invalidLog3.txt | 5 + .../src/test/resources/trace/invalidLog4.txt | 5 + .../src/test/resources/trace/invalidLog5.txt | 7 + .../src/test/resources/trace/invalidLog6.txt | 5 + .../src/test/resources/trace/invalidLog7.txt | 5 + .../src/test/resources/trace/invalidLog8.txt | 6 + .../src/test/resources/trace/invalidLog9.txt | 6 + .../src/test/resources/trace/nativeLog.txt | 1 + .../src/test/resources/trace/rmLog.txt | 30 + hadoop-tools/hadoop-tools-dist/pom.xml | 6 + hadoop-tools/pom.xml | 1 + .../Apache_Hadoop_YARN_Client_2.8.2.xml | 2316 + .../Apache_Hadoop_YARN_Common_2.8.2.xml | 2665 ++ .../Apache_Hadoop_YARN_Server_Common_2.8.2.xml | 829 + .../RegisterApplicationMasterResponse.java | 31 + .../yarn/api/records/ReservationDefinition.java | 12 +- .../yarn/api/records/ResourceInformation.java | 6 + .../hadoop/yarn/conf/YarnConfiguration.java | 18 + .../yarn/util/resource/ResourceUtils.java | 23 +- .../src/main/proto/yarn_service_protos.proto | 1 + .../yarn/client/api/impl/YarnClientImpl.java | 16 + ...RegisterApplicationMasterResponsePBImpl.java | 79 +- .../impl/pb/ReservationDefinitionPBImpl.java | 5 + .../resource/DominantResourceCalculator.java | 10 +- .../src/main/resources/yarn-default.xml | 26 + .../impl/pb/NodeHeartbeatResponsePBImpl.java | 11 +- .../LocalityMulticastAMRMProxyPolicy.java | 40 +- .../TestLocalityMulticastAMRMProxyPolicy.java | 57 + .../resourcemanager/DefaultAMSProcessor.java | 2 + .../resourcemanager/recovery/RMStateStore.java | 2 + .../recovery/ZKRMStateStore.java | 411 +- .../reservation/CapacityOverTimePolicy.java | 3 - .../reservation/InMemoryPlan.java | 48 +- .../reservation/PlanContext.java | 11 + .../reservation/ReservationInputValidator.java | 8 + .../scheduler/fair/FSAppAttempt.java | 10 + .../resourcemanager/webapp/RMWebServices.java | 14 +- .../resourcemanager/webapp/dao/AppInfo.java | 4 +- .../webapp/dao/ReservationDefinitionInfo.java | 12 + .../resourcemanager/TestResourceManager.java | 9 +- .../TestWorkPreservingRMRestart.java | 21 +- .../recovery/TestZKRMStateStore.java | 372 +- .../reservation/TestInMemoryPlan.java | 261 + .../TestReservationInputValidator.java | 23 + .../conf/ConfigurationStoreBaseTest.java | 4 + .../conf/TestLeveldbConfigurationStore.java | 1 - .../capacity/conf/TestZKConfigurationStore.java | 3 +- .../scheduler/fair/TestFSLeafQueue.java | 1 + .../scheduler/fair/TestFairScheduler.java | 2 + .../webapp/TestRMWebServicesReservation.java | 68 +- .../src/test/resources/submit-reservation.json | 1 + .../src/site/markdown/FairScheduler.md | 24 + .../src/site/markdown/ReservationSystem.md | 2 +- .../src/site/markdown/ResourceManagerRest.md | 1 + pom.xml | 2 +- 192 files changed, 81469 insertions(+), 428 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d11161bb/LICENSE.txt ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d11161bb/dev-support/docker/Dockerfile ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d11161bb/hadoop-project/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d11161bb/hadoop-project/src/site/site.xml ---------------------------------------------------------------------- diff --cc hadoop-project/src/site/site.xml index a8bbe20,6fc3cf9..39a32f2 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@@ -103,17 -103,9 +103,18 @@@ <item name="Disk Balancer" href="hadoop-project-dist/hadoop-hdfs/HDFSDiskbalancer.html"/> <item name="Upgrade Domain" href="hadoop-project-dist/hadoop-hdfs/HdfsUpgradeDomain.html"/> <item name="DataNode Admin" href="hadoop-project-dist/hadoop-hdfs/HdfsDataNodeAdminGuide.html"/> + <item name="Router Federation" href="hadoop-project-dist/hadoop-hdfs/HDFSRouterFederation.html"/> </menu> + <menu name="Ozone" inherit="top"> + <item name="Getting Started" href="hadoop-project-dist/hadoop-hdfs/OzoneGettingStarted.html"/> + <item name="Ozone Overview" + href="hadoop-project-dist/hadoop-hdfs/OzoneOverview.html"/> + <item name="Commands Reference" href="hadoop-project-dist/hadoop-hdfs/OzoneCommandShell.html"/> + <item name="Ozone Rest API" href="hadoop-project-dist/hadoop-hdfs/OzoneRest.html"/> + <item name="Ozone Metrics" href="hadoop-project-dist/hadoop-hdfs/OzoneMetrics.html"/> + </menu> + <menu name="MapReduce" inherit="top"> <item name="Tutorial" href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapReduceTutorial.html"/> <item name="Commands Reference" href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapredCommands.html"/> http://git-wip-us.apache.org/repos/asf/hadoop/blob/d11161bb/hadoop-tools/hadoop-tools-dist/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d11161bb/hadoop-tools/pom.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
