Merge remote-tracking branch 'apache/trunk' into YARN-2928
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a3d0af5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a3d0af5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a3d0af5 Branch: refs/heads/YARN-2928 Commit: 5a3d0af568c4410428e5d6022118e9b1e1475de5 Parents: e278615 2aa9979 Author: Zhijie Shen <[email protected]> Authored: Wed Feb 18 13:18:29 2015 -0800 Committer: Zhijie Shen <[email protected]> Committed: Wed Feb 18 13:18:29 2015 -0800 ---------------------------------------------------------------------- dev-support/smart-apply-patch.sh | 4 +- .../client/KerberosAuthenticator.java | 26 +- .../server/AuthenticationToken.java | 162 +- .../security/authentication/util/AuthToken.java | 218 ++ .../hadoop-auth/src/site/apt/BuildingIt.apt.vm | 70 - .../src/site/apt/Configuration.apt.vm | 377 --- .../hadoop-auth/src/site/apt/Examples.apt.vm | 133 - .../hadoop-auth/src/site/apt/index.apt.vm | 59 - .../hadoop-auth/src/site/markdown/BuildingIt.md | 56 + .../src/site/markdown/Configuration.md | 341 +++ .../hadoop-auth/src/site/markdown/Examples.md | 109 + .../hadoop-auth/src/site/markdown/index.md | 43 + .../server/TestAuthenticationToken.java | 100 - .../authentication/util/TestAuthToken.java | 127 + hadoop-common-project/hadoop-common/CHANGES.txt | 64 +- .../hadoop-common/src/main/bin/hadoop | 5 + .../src/main/bin/hadoop-config.cmd | 4 +- .../hadoop-common/src/main/bin/hadoop-config.sh | 9 +- .../hadoop-common/src/main/bin/hadoop-daemon.sh | 3 + .../src/main/bin/hadoop-daemons.sh | 16 +- .../src/main/bin/hadoop-functions.sh | 14 + .../org/apache/hadoop/conf/Configuration.java | 6 +- .../java/org/apache/hadoop/fs/CanUnbuffer.java | 36 + .../org/apache/hadoop/fs/FSDataInputStream.java | 12 +- .../apache/hadoop/fs/shell/XAttrCommands.java | 10 +- .../main/java/org/apache/hadoop/ha/HAAdmin.java | 4 +- .../org/apache/hadoop/ha/HealthMonitor.java | 35 +- .../main/java/org/apache/hadoop/ipc/Client.java | 3 +- .../main/java/org/apache/hadoop/ipc/Server.java | 1 + .../java/org/apache/hadoop/log/LogLevel.java | 10 +- .../metrics/spi/AbstractMetricsContext.java | 12 +- .../java/org/apache/hadoop/net/NetUtils.java | 1 + .../hadoop/security/alias/CredentialShell.java | 19 +- .../src/main/resources/core-default.xml | 77 +- .../src/site/markdown/ClusterSetup.md | 2 +- .../src/site/markdown/CommandsManual.md | 10 +- .../src/site/markdown/SecureMode.md | 44 +- .../hadoop-common/src/site/markdown/Tracing.md | 55 +- .../apache/hadoop/conf/TestConfiguration.java | 29 +- .../org/apache/hadoop/ha/DummyHAService.java | 73 +- .../org/apache/hadoop/ha/TestHealthMonitor.java | 4 +- .../java/org/apache/hadoop/io/TestMapFile.java | 5 +- .../java/org/apache/hadoop/ipc/TestIPC.java | 6 + .../java/org/apache/hadoop/ipc/TestRPC.java | 68 + .../org/apache/hadoop/net/TestNetUtils.java | 11 + .../hadoop/security/alias/TestCredShell.java | 15 + .../hadoop-kms/src/site/apt/index.apt.vm | 1020 ------- .../hadoop-kms/src/site/markdown/index.md.vm | 864 ++++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 109 +- .../bkjournal/BookKeeperJournalManager.java | 47 +- .../bkjournal/TestBookKeeperJournalManager.java | 153 +- .../TestBookKeeperSpeculativeRead.java | 167 ++ .../hadoop-hdfs/src/main/bin/hdfs | 5 + .../java/org/apache/hadoop/hdfs/DFSClient.java | 77 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 + .../org/apache/hadoop/hdfs/DFSInputStream.java | 30 +- .../org/apache/hadoop/hdfs/DFSOutputStream.java | 39 +- .../hadoop/hdfs/DistributedFileSystem.java | 60 +- .../java/org/apache/hadoop/hdfs/PeerCache.java | 9 +- .../apache/hadoop/hdfs/client/HdfsAdmin.java | 18 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 8 +- .../hdfs/protocol/DatanodeInfoWithStorage.java | 4 + .../hadoop/hdfs/protocol/LocatedBlock.java | 47 +- .../datatransfer/DataTransferProtocol.java | 6 +- .../hdfs/protocol/datatransfer/Receiver.java | 4 +- .../hdfs/protocol/datatransfer/Sender.java | 8 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 2 +- .../ClientNamenodeProtocolTranslatorPB.java | 4 +- .../DatanodeProtocolClientSideTranslatorPB.java | 9 +- .../DatanodeProtocolServerSideTranslatorPB.java | 7 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 46 + .../hadoop/hdfs/server/balancer/Balancer.java | 7 +- .../blockmanagement/DatanodeDescriptor.java | 39 +- .../server/blockmanagement/DatanodeManager.java | 19 +- .../blockmanagement/HeartbeatManager.java | 8 +- .../hdfs/server/datanode/BPOfferService.java | 8 +- .../hdfs/server/datanode/BPServiceActor.java | 69 +- .../server/datanode/BPServiceActorAction.java | 32 + .../datanode/BPServiceActorActionException.java | 33 + .../hdfs/server/datanode/BlockReceiver.java | 12 +- .../hdfs/server/datanode/BlockScanner.java | 32 + .../hdfs/server/datanode/BlockSender.java | 3 + .../hadoop/hdfs/server/datanode/DataNode.java | 8 +- .../hdfs/server/datanode/DataXceiver.java | 34 +- .../hdfs/server/datanode/DataXceiverServer.java | 4 +- .../hdfs/server/datanode/ErrorReportAction.java | 83 + .../server/datanode/ReportBadBlockAction.java | 108 + .../hdfs/server/datanode/VolumeScanner.java | 133 +- .../server/datanode/fsdataset/FsDatasetSpi.java | 21 + .../datanode/fsdataset/impl/FsDatasetImpl.java | 148 +- .../datanode/fsdataset/impl/FsVolumeList.java | 41 +- .../fsdataset/impl/VolumeFailureInfo.java | 82 + .../server/datanode/metrics/FSDatasetMBean.java | 19 + .../server/datanode/web/DatanodeHttpServer.java | 2 +- .../datanode/web/webhdfs/ParameterParser.java | 2 +- .../hadoop/hdfs/server/namenode/BackupNode.java | 6 +- .../namenode/DirectoryWithQuotaFeature.java | 111 +- .../hdfs/server/namenode/FSDirAttrOp.java | 36 +- .../hdfs/server/namenode/FSDirConcatOp.java | 2 +- .../hdfs/server/namenode/FSDirectory.java | 39 +- .../hadoop/hdfs/server/namenode/FSImage.java | 16 +- .../hdfs/server/namenode/FSImageFormat.java | 9 +- .../server/namenode/FSImageFormatPBINode.java | 6 +- .../server/namenode/FSImageSerialization.java | 2 +- .../hdfs/server/namenode/FSNamesystem.java | 65 +- .../hadoop/hdfs/server/namenode/INode.java | 18 +- .../hdfs/server/namenode/INodeDirectory.java | 12 +- .../namenode/INodeDirectoryAttributes.java | 8 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 46 +- .../hdfs/server/namenode/INodeReference.java | 4 +- .../hdfs/server/namenode/INodeSymlink.java | 2 +- .../hdfs/server/namenode/ImageServlet.java | 2 +- .../hadoop/hdfs/server/namenode/NameNode.java | 4 +- .../server/namenode/NameNodeHttpServer.java | 4 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 10 +- .../hadoop/hdfs/server/namenode/Quota.java | 10 +- .../hdfs/server/namenode/QuotaCounts.java | 98 +- .../hdfs/server/namenode/SecondaryNameNode.java | 4 +- .../namenode/metrics/FSNamesystemMBean.java | 13 + .../snapshot/FileWithSnapshotFeature.java | 14 +- .../namenode/snapshot/SnapshotDiffInfo.java | 9 +- .../hdfs/server/protocol/DatanodeProtocol.java | 5 +- .../server/protocol/VolumeFailureSummary.java | 72 + .../offlineImageViewer/PBImageTextWriter.java | 2 +- .../hadoop/hdfs/web/URLConnectionFactory.java | 7 +- .../src/main/native/fuse-dfs/fuse_connect.c | 16 + .../hadoop-hdfs/src/main/native/libhdfs/hdfs.c | 28 + .../hadoop-hdfs/src/main/native/libhdfs/hdfs.h | 9 + .../src/main/proto/ClientNamenodeProtocol.proto | 2 +- .../src/main/proto/DatanodeProtocol.proto | 17 +- .../src/main/proto/datatransfer.proto | 3 + .../src/main/resources/hdfs-default.xml | 6 + .../src/main/webapps/hdfs/dfshealth.html | 33 + .../src/main/webapps/hdfs/dfshealth.js | 43 + .../src/main/webapps/hdfs/explorer.js | 7 + .../src/site/apt/ArchivalStorage.apt.vm | 233 -- .../site/apt/CentralizedCacheManagement.apt.vm | 344 --- .../src/site/apt/ExtendedAttributes.apt.vm | 97 - .../src/site/apt/FaultInjectFramework.apt.vm | 312 -- .../hadoop-hdfs/src/site/apt/Federation.apt.vm | 339 --- .../src/site/apt/HDFSCommands.apt.vm | 797 ------ .../site/apt/HDFSHighAvailabilityWithNFS.apt.vm | 859 ------ .../site/apt/HDFSHighAvailabilityWithQJM.apt.vm | 816 ------ .../hadoop-hdfs/src/site/apt/HdfsDesign.apt.vm | 510 ---- .../src/site/apt/HdfsEditsViewer.apt.vm | 104 - .../src/site/apt/HdfsImageViewer.apt.vm | 247 -- .../src/site/apt/HdfsMultihoming.apt.vm | 145 - .../src/site/apt/HdfsNfsGateway.apt.vm | 364 --- .../src/site/apt/HdfsPermissionsGuide.apt.vm | 438 --- .../src/site/apt/HdfsQuotaAdminGuide.apt.vm | 116 - .../src/site/apt/HdfsUserGuide.apt.vm | 556 ---- .../hadoop-hdfs/src/site/apt/LibHdfs.apt.vm | 101 - .../src/site/apt/SLGUserGuide.apt.vm | 195 -- .../src/site/apt/ShortCircuitLocalReads.apt.vm | 112 - .../src/site/apt/TransparentEncryption.apt.vm | 290 -- .../hadoop-hdfs/src/site/apt/ViewFs.apt.vm | 304 -- .../hadoop-hdfs/src/site/apt/WebHDFS.apt.vm | 2628 ----------------- .../src/site/markdown/ArchivalStorage.md | 160 ++ .../site/markdown/CentralizedCacheManagement.md | 268 ++ .../src/site/markdown/ExtendedAttributes.md | 98 + .../src/site/markdown/FaultInjectFramework.md | 254 ++ .../hadoop-hdfs/src/site/markdown/Federation.md | 254 ++ .../src/site/markdown/HDFSCommands.md | 514 ++++ .../markdown/HDFSHighAvailabilityWithNFS.md | 678 +++++ .../markdown/HDFSHighAvailabilityWithQJM.md | 642 +++++ .../hadoop-hdfs/src/site/markdown/HdfsDesign.md | 240 ++ .../src/site/markdown/HdfsEditsViewer.md | 69 + .../src/site/markdown/HdfsImageViewer.md | 172 ++ .../src/site/markdown/HdfsMultihoming.md | 127 + .../src/site/markdown/HdfsNfsGateway.md | 254 ++ .../src/site/markdown/HdfsPermissionsGuide.md | 284 ++ .../src/site/markdown/HdfsQuotaAdminGuide.md | 93 + .../src/site/markdown/HdfsUserGuide.md | 375 +++ .../hadoop-hdfs/src/site/markdown/LibHdfs.md | 92 + .../src/site/markdown/SLGUserGuide.md | 157 + .../src/site/markdown/ShortCircuitLocalReads.md | 87 + .../src/site/markdown/TransparentEncryption.md | 268 ++ .../hadoop-hdfs/src/site/markdown/ViewFs.md | 242 ++ .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 1939 +++++++++++++ .../java/org/apache/hadoop/fs/TestUnbuffer.java | 127 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 27 +- .../hadoop/hdfs/TestDataTransferProtocol.java | 2 +- .../hdfs/server/balancer/TestBalancer.java | 71 +- .../blockmanagement/TestBlockManager.java | 3 +- .../TestNameNodePrunesMissingStorages.java | 4 +- .../TestOverReplicatedBlocks.java | 2 +- .../blockmanagement/TestReplicationPolicy.java | 2 +- .../TestReplicationPolicyConsiderLoad.java | 8 +- .../TestReplicationPolicyWithNodeGroup.java | 2 +- .../server/datanode/SimulatedFSDataset.java | 33 + .../server/datanode/TestBPOfferService.java | 161 +- .../hdfs/server/datanode/TestBlockRecovery.java | 4 +- .../hdfs/server/datanode/TestBlockScanner.java | 131 + .../TestDataNodeVolumeFailureReporting.java | 421 ++- .../hdfs/server/datanode/TestDiskError.java | 2 +- .../server/datanode/TestFsDatasetCache.java | 3 +- .../hdfs/server/datanode/TestStorageReport.java | 4 +- .../extdataset/ExternalDatasetImpl.java | 30 + .../fsdataset/impl/FsVolumeListTest.java | 101 - .../fsdataset/impl/TestFsDatasetImpl.java | 6 +- .../fsdataset/impl/TestFsVolumeList.java | 104 + .../web/webhdfs/TestParameterParser.java | 13 + .../server/namenode/NNThroughputBenchmark.java | 4 +- .../hdfs/server/namenode/NameNodeAdapter.java | 2 +- .../hdfs/server/namenode/TestDeadDatanode.java | 2 +- .../namenode/TestDiskspaceQuotaUpdate.java | 14 +- .../namenode/TestFavoredNodesEndToEnd.java | 29 + .../hdfs/server/namenode/TestMalformedURLs.java | 59 + .../server/namenode/TestQuotaByStorageType.java | 193 +- .../snapshot/TestRenameWithSnapshots.java | 10 +- .../namenode/snapshot/TestSnapshotDeletion.java | 4 +- .../src/test/resources/hdfs-site.malformed.xml | 143 + hadoop-mapreduce-project/CHANGES.txt | 34 + hadoop-mapreduce-project/bin/mapred | 5 + .../bin/mr-jobhistory-daemon.sh | 3 + .../jobhistory/JobHistoryEventHandler.java | 7 +- .../mapreduce/v2/app/webapp/CountersBlock.java | 6 +- .../org/apache/hadoop/mapred/IndexCache.java | 2 +- .../java/org/apache/hadoop/mapred/MapTask.java | 4 + .../org/apache/hadoop/mapred/QueueManager.java | 4 +- .../apache/hadoop/mapred/TaskLogAppender.java | 2 +- .../mapred/lib/CombineFileRecordReader.java | 5 - .../java/org/apache/hadoop/mapreduce/Job.java | 47 - .../apache/hadoop/mapreduce/JobSubmitter.java | 15 +- .../hadoop/mapreduce/lib/chain/Chain.java | 5 +- .../lib/fieldsel/FieldSelectionHelper.java | 2 +- .../lib/input/CombineFileRecordReader.java | 5 - .../mapreduce/security/SecureShuffleUtils.java | 15 +- .../mapreduce/task/reduce/InMemoryReader.java | 4 +- .../task/reduce/ShuffleSchedulerImpl.java | 9 +- .../org/apache/hadoop/mapreduce/tools/CLI.java | 21 +- .../hadoop/mapreduce/util/ResourceBundles.java | 2 +- .../src/site/apt/DistributedCacheDeploy.apt.vm | 151 - .../src/site/apt/EncryptedShuffle.apt.vm | 320 --- .../src/site/apt/MapReduceTutorial.apt.vm | 1605 ----------- ...pReduce_Compatibility_Hadoop1_Hadoop2.apt.vm | 114 - .../src/site/apt/MapredAppMasterRest.apt.vm | 2709 ------------------ .../src/site/apt/MapredCommands.apt.vm | 233 -- .../apt/PluggableShuffleAndPluggableSort.apt.vm | 98 - .../site/markdown/DistributedCacheDeploy.md.vm | 119 + .../src/site/markdown/EncryptedShuffle.md | 255 ++ .../src/site/markdown/MapReduceTutorial.md | 1156 ++++++++ .../MapReduce_Compatibility_Hadoop1_Hadoop2.md | 69 + .../src/site/markdown/MapredAppMasterRest.md | 2397 ++++++++++++++++ .../src/site/markdown/MapredCommands.md | 153 + .../PluggableShuffleAndPluggableSort.md | 73 + .../apache/hadoop/mapreduce/tools/TestCLI.java | 45 + .../hadoop/mapreduce/v2/hs/CompletedJob.java | 8 +- .../src/site/apt/HistoryServerRest.apt.vm | 2672 ----------------- .../src/site/markdown/HistoryServerRest.md | 2361 +++++++++++++++ .../TestClientProtocolProviderImpls.java | 71 +- hadoop-project/src/site/apt/index.apt.vm | 73 - hadoop-project/src/site/markdown/index.md.vm | 72 + .../org/apache/hadoop/fs/s3a/Constants.java | 4 + .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 + .../apache/hadoop/fs/s3a/S3AInputStream.java | 20 +- .../src/site/markdown/tools/hadoop-aws/index.md | 83 +- hadoop-tools/hadoop-distcp/README | 2 - .../src/site/markdown/DistCp.md.vm | 4 +- .../hadoop/mapred/gridmix/TestHighRamJob.java | 11 +- .../hadoop-openstack/src/site/apt/index.apt.vm | 686 ----- .../hadoop-openstack/src/site/markdown/index.md | 544 ++++ .../src/site/resources/css/site.css | 30 + .../src/site/apt/SchedulerLoadSimulator.apt.vm | 439 --- .../src/site/markdown/SchedulerLoadSimulator.md | 357 +++ .../src/site/apt/HadoopStreaming.apt.vm | 792 ----- .../src/site/markdown/HadoopStreaming.md.vm | 559 ++++ hadoop-yarn-project/CHANGES.txt | 65 + hadoop-yarn-project/hadoop-yarn/bin/yarn | 5 + .../hadoop-yarn/bin/yarn-daemon.sh | 3 + .../hadoop-yarn/bin/yarn-daemons.sh | 19 +- .../dev-support/findbugs-exclude.xml | 29 +- .../hadoop/yarn/api/records/ContainerId.java | 5 + .../hadoop/yarn/client/cli/RMAdminCLI.java | 29 +- .../hadoop/yarn/client/cli/TestRMAdminCLI.java | 18 +- .../nodelabels/CommonNodeLabelsManager.java | 7 +- .../apache/hadoop/yarn/util/ConverterUtils.java | 7 + .../src/main/resources/yarn-default.xml | 5 +- .../hadoop/yarn/util/TestConverterUtils.java | 15 + .../server/timeline/LeveldbTimelineStore.java | 2 +- .../server/timeline/TimelineStoreTestUtils.java | 3 + .../containermanager/AuxServices.java | 2 +- .../containermanager/ContainerManagerImpl.java | 4 +- .../application/ApplicationImpl.java | 2 +- .../container/ContainerImpl.java | 2 +- .../logaggregation/AppLogAggregatorImpl.java | 15 + .../loghandler/NonAggregatingLogHandler.java | 63 +- .../recovery/NMLeveldbStateStoreService.java | 67 +- .../recovery/NMNullStateStoreService.java | 16 + .../recovery/NMStateStoreService.java | 35 + .../impl/container-executor.c | 3 +- .../test/test-container-executor.c | 24 +- .../yarn_server_nodemanager_recovery.proto | 4 + .../TestDockerContainerExecutorWithMocks.java | 6 +- .../TestNonAggregatingLogHandler.java | 79 +- .../recovery/NMMemoryStateStoreService.java | 79 +- .../TestNMLeveldbStateStoreService.java | 51 + .../ApplicationMasterService.java | 16 +- .../resourcemanager/ResourceTrackerService.java | 4 +- .../amlauncher/ApplicationMasterLauncher.java | 1 + .../nodelabels/RMNodeLabelsManager.java | 20 +- .../rmapp/attempt/RMAppAttemptImpl.java | 52 +- .../scheduler/AppSchedulingInfo.java | 3 +- .../scheduler/capacity/AbstractCSQueue.java | 240 +- .../scheduler/capacity/CSQueue.java | 49 +- .../scheduler/capacity/CSQueueUtils.java | 160 +- .../CapacitySchedulerConfiguration.java | 73 +- .../scheduler/capacity/LeafQueue.java | 196 +- .../scheduler/capacity/ParentQueue.java | 92 +- .../scheduler/capacity/PlanQueue.java | 13 +- .../scheduler/capacity/QueueCapacities.java | 68 +- .../scheduler/capacity/ReservationQueue.java | 10 +- .../scheduler/fair/AllocationConfiguration.java | 13 +- .../fair/AllocationFileLoaderService.java | 2 +- .../scheduler/fair/FSOpDurations.java | 3 + .../yarn/server/resourcemanager/MockAM.java | 28 +- .../server/resourcemanager/TestRMRestart.java | 4 + .../nodelabels/TestRMNodeLabelsManager.java | 63 +- .../attempt/TestRMAppAttemptTransitions.java | 2 +- .../scheduler/capacity/TestCSQueueUtils.java | 24 +- .../capacity/TestCapacityScheduler.java | 6 +- .../TestCapacitySchedulerNodeLabelUpdate.java | 3 +- .../scheduler/capacity/TestQueueCapacities.java | 2 +- .../scheduler/capacity/TestQueueParsing.java | 14 +- .../security/TestAMRMTokens.java | 45 +- .../webapp/TestRMWebServicesCapacitySched.java | 4 +- .../yarn/server/webproxy/ProxyUriUtils.java | 11 +- .../hadoop/yarn/server/webproxy/ProxyUtils.java | 127 + .../yarn/server/webproxy/WebAppProxy.java | 14 +- .../yarn/server/webproxy/WebAppProxyServer.java | 20 +- .../server/webproxy/WebAppProxyServlet.java | 131 +- .../webproxy/amfilter/AmFilterInitializer.java | 2 +- .../server/webproxy/amfilter/AmIpFilter.java | 48 +- .../server/webproxy/TestWebAppProxyServlet.java | 84 +- .../server/webproxy/amfilter/TestAmFilter.java | 39 +- .../src/site/apt/ResourceManagerRestart.apt.vm | 182 +- .../src/site/apt/YarnCommands.apt.vm | 21 +- 337 files changed, 23330 insertions(+), 23584 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3d0af5/hadoop-yarn-project/CHANGES.txt ----------------------------------------------------------------------
