Merge remote-tracking branch 'apache/trunk' into YARN-2928
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5de4026d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5de4026d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5de4026d Branch: refs/heads/YARN-2928 Commit: 5de4026d8aef6c3343d24aa3831da48ad7a1a87e Parents: fb1b596 3bc72cc Author: Zhijie Shen <[email protected]> Authored: Tue Mar 17 20:22:11 2015 -0700 Committer: Zhijie Shen <[email protected]> Committed: Tue Mar 17 20:22:11 2015 -0700 ---------------------------------------------------------------------- dev-support/test-patch.sh | 4 + hadoop-common-project/hadoop-common/CHANGES.txt | 36 +- .../org/apache/hadoop/crypto/CryptoCodec.java | 10 +- .../hadoop/crypto/CryptoOutputStream.java | 19 +- .../fs/CommonConfigurationKeysPublic.java | 11 + .../ZKDelegationTokenSecretManager.java | 21 +- .../apache/hadoop/tracing/SpanReceiverHost.java | 13 +- .../hadoop/crypto/random/OpensslSecureRandom.c | 18 +- .../crypto/TestCryptoStreamsForLocalFS.java | 8 +- ...stCryptoStreamsWithJceAesCtrCryptoCodec.java | 38 ++ ...yptoStreamsWithOpensslAesCtrCryptoCodec.java | 7 + hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 29 +- .../src/contrib/libwebhdfs/src/hdfs_web.c | 6 + .../apache/hadoop/hdfs/BlockReaderFactory.java | 23 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 5 + .../org/apache/hadoop/hdfs/DFSOutputStream.java | 9 +- .../hadoop/hdfs/protocol/BlockListAsLongs.java | 660 +++++++++++-------- .../datatransfer/DataTransferProtocol.java | 5 +- .../hdfs/protocol/datatransfer/Receiver.java | 2 +- .../hdfs/protocol/datatransfer/Sender.java | 4 +- .../DatanodeProtocolClientSideTranslatorPB.java | 22 +- .../DatanodeProtocolServerSideTranslatorPB.java | 14 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 6 +- .../BlockInfoContiguousUnderConstruction.java | 1 + .../server/blockmanagement/BlockManager.java | 16 +- .../hdfs/server/datanode/BPServiceActor.java | 13 +- .../hdfs/server/datanode/DataXceiver.java | 95 +-- .../server/datanode/ShortCircuitRegistry.java | 13 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 20 +- .../hdfs/server/namenode/FSNamesystem.java | 15 + .../hdfs/server/namenode/NameNodeRpcServer.java | 2 +- .../server/namenode/snapshot/FileDiffList.java | 19 +- .../server/protocol/DatanodeRegistration.java | 9 + .../hdfs/server/protocol/NamespaceInfo.java | 52 ++ .../server/protocol/StorageBlockReport.java | 8 +- .../hadoop-hdfs/src/main/native/libhdfs/hdfs.c | 37 ++ .../hadoop-hdfs/src/main/native/libhdfs/hdfs.h | 15 + .../src/main/proto/DatanodeProtocol.proto | 2 + .../src/main/proto/datatransfer.proto | 11 + .../hadoop-hdfs/src/main/proto/hdfs.proto | 1 + .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 76 ++- .../apache/hadoop/hdfs/TestFileCreation.java | 4 +- .../hdfs/protocol/TestBlockListAsLongs.java | 237 +++++++ .../blockmanagement/TestBlockManager.java | 8 +- .../server/datanode/BlockReportTestBase.java | 27 +- .../server/datanode/SimulatedFSDataset.java | 11 +- .../TestBlockHasMultipleReplicasOnSameDN.java | 9 +- .../datanode/TestDataNodeVolumeFailure.java | 4 +- ...TestDnRespectsBlockReportSplitThreshold.java | 2 +- .../extdataset/ExternalDatasetImpl.java | 2 +- .../server/namenode/NNThroughputBenchmark.java | 23 +- .../hdfs/server/namenode/TestDeadDatanode.java | 3 +- .../hdfs/server/namenode/TestFSImage.java | 2 + .../hdfs/server/namenode/TestFileTruncate.java | 50 +- .../snapshot/TestRenameWithSnapshots.java | 4 +- .../shortcircuit/TestShortCircuitCache.java | 63 ++ .../TestOfflineEditsViewer.java | 9 +- hadoop-mapreduce-project/CHANGES.txt | 23 +- .../v2/app/launcher/ContainerLauncherImpl.java | 14 +- .../jobhistory/TestJobHistoryEventHandler.java | 14 +- .../v2/app/launcher/TestContainerLauncher.java | 21 +- .../java/org/apache/hadoop/mapred/JobConf.java | 5 + .../java/org/apache/hadoop/mapred/MapTask.java | 2 + .../apache/hadoop/mapreduce/JobSubmitter.java | 2 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 9 + .../mapreduce/task/reduce/MergeManagerImpl.java | 5 +- .../src/main/resources/mapred-default.xml | 8 + .../mapred/TestMRTimelineEventHandling.java | 12 +- .../java/org/apache/hadoop/ant/DfsTask.java | 6 +- .../org/apache/hadoop/fs/s3/S3FileSystem.java | 4 +- .../hadoop/fs/s3a/S3AFastOutputStream.java | 4 +- .../hadoop/fs/s3native/NativeS3FileSystem.java | 24 +- .../fs/azure/AzureNativeFileSystemStore.java | 22 +- .../hadoop/fs/azure/NativeAzureFileSystem.java | 16 +- .../hadoop/tools/CopyListingFileStatus.java | 8 +- .../apache/hadoop/tools/SimpleCopyListing.java | 2 +- .../apache/hadoop/tools/util/DistCpUtils.java | 4 +- .../mapred/gridmix/TestRandomAlgorithm.java | 2 - .../src/site/markdown/SchedulerLoadSimulator.md | 2 +- .../java/org/apache/hadoop/record/Buffer.java | 8 +- .../java/org/apache/hadoop/record/Utils.java | 8 +- .../src/site/markdown/HadoopStreaming.md.vm | 14 +- hadoop-yarn-project/CHANGES.txt | 32 +- .../dev-support/findbugs-exclude.xml | 27 + .../yarn/api/ApplicationBaseProtocol.java | 44 +- .../yarn/api/ApplicationClientProtocol.java | 3 - .../api/protocolrecords/AllocateRequest.java | 25 +- .../api/protocolrecords/AllocateResponse.java | 68 +- .../FinishApplicationMasterRequest.java | 25 +- .../FinishApplicationMasterResponse.java | 7 +- .../protocolrecords/GetApplicationsRequest.java | 1 - .../GetClusterMetricsResponse.java | 4 +- .../GetContainerStatusesRequest.java | 2 - .../GetContainerStatusesResponse.java | 2 - .../protocolrecords/GetQueueInfoRequest.java | 2 +- .../protocolrecords/GetQueueInfoResponse.java | 11 +- .../KillApplicationResponse.java | 9 +- .../RegisterApplicationMasterRequest.java | 33 +- .../RegisterApplicationMasterResponse.java | 11 +- .../protocolrecords/StartContainerRequest.java | 9 +- .../api/records/ApplicationAttemptReport.java | 23 +- .../yarn/api/records/ApplicationReport.java | 47 +- .../records/ApplicationSubmissionContext.java | 50 +- .../hadoop/yarn/api/records/Container.java | 49 +- .../api/records/ContainerLaunchContext.java | 35 +- .../yarn/api/records/ContainerReport.java | 29 +- .../yarn/api/records/ContainerStatus.java | 21 +- .../yarn/api/records/LocalResourceType.java | 32 +- .../api/records/LocalResourceVisibility.java | 31 +- .../yarn/api/records/LogAggregationContext.java | 39 +- .../hadoop/yarn/api/records/NodeReport.java | 25 +- .../yarn/api/records/PreemptionMessage.java | 32 +- .../hadoop/yarn/api/records/QueueACL.java | 13 +- .../hadoop/yarn/api/records/QueueInfo.java | 25 +- .../hadoop/yarn/api/records/QueueState.java | 15 +- .../yarn/api/records/ReservationRequest.java | 17 +- .../records/ReservationRequestInterpreter.java | 38 +- .../yarn/api/records/ResourceRequest.java | 51 +- .../hadoop/yarn/conf/YarnConfiguration.java | 10 +- .../UpdateNodeResourceRequest.java | 4 +- .../distributedshell/TestDistributedShell.java | 4 +- .../hadoop/yarn/client/api/AHSClient.java | 24 +- .../hadoop/yarn/client/api/AMRMClient.java | 4 +- .../apache/hadoop/yarn/client/api/NMClient.java | 4 +- .../hadoop/yarn/client/api/NMTokenCache.java | 58 +- .../hadoop/yarn/client/api/YarnClient.java | 23 +- .../nodelabels/CommonNodeLabelsManager.java | 6 +- .../hadoop/yarn/nodelabels/NodeLabelsStore.java | 3 +- .../server/security/ApplicationACLsManager.java | 1 - .../apache/hadoop/yarn/util/StringHelper.java | 6 +- .../org/apache/hadoop/yarn/webapp/WebApps.java | 4 +- .../registry/client/binding/RegistryUtils.java | 8 +- .../client/impl/RegistryOperationsClient.java | 2 +- .../client/impl/zk/ZookeeperConfigOptions.java | 3 +- .../server/services/MicroZookeeperService.java | 10 +- .../registry/server/services/package-info.java | 9 +- .../server/timeline/LeveldbTimelineStore.java | 18 +- .../server/timeline/MemoryTimelineStore.java | 12 +- .../server/timeline/TimelineDataManager.java | 50 +- .../yarn/server/timeline/TimelineReader.java | 3 +- ...TimelineAuthenticationFilterInitializer.java | 13 +- .../timeline/TestLeveldbTimelineStore.java | 16 +- .../timeline/TestTimelineDataManager.java | 26 +- .../server/timeline/TimelineStoreTestUtils.java | 33 +- .../org/apache/hadoop/yarn/lib/ZKClient.java | 2 +- .../RegisterNodeManagerRequest.java | 3 +- .../server/api/records/NodeHealthStatus.java | 24 +- .../hadoop/yarn/server/webapp/WebPageUtils.java | 6 +- .../server/nodemanager/ContainerExecutor.java | 8 +- .../util/NodeManagerHardwareUtils.java | 8 +- .../TestDockerContainerExecutor.java | 2 +- .../server/resourcemanager/RMAppManager.java | 6 +- .../recovery/FileSystemRMStateStore.java | 22 +- .../rmapp/attempt/RMAppAttempt.java | 11 +- .../scheduler/SchedulerNode.java | 2 +- .../scheduler/SchedulerUtils.java | 3 +- .../scheduler/capacity/AbstractCSQueue.java | 112 +++- .../scheduler/capacity/CSQueue.java | 4 +- .../scheduler/capacity/CapacityScheduler.java | 38 +- .../scheduler/capacity/LeafQueue.java | 292 +++----- .../scheduler/capacity/ParentQueue.java | 140 +--- .../scheduler/common/fica/FiCaSchedulerApp.java | 16 +- .../scheduler/fair/AllocationConfiguration.java | 13 +- .../fair/AllocationFileLoaderService.java | 2 +- .../scheduler/fair/FSOpDurations.java | 3 - .../fair/policies/ComputeFairShares.java | 19 +- .../security/DelegationTokenRenewer.java | 2 - .../server/resourcemanager/TestAppManager.java | 5 + .../resourcemanager/TestClientRMService.java | 5 + .../recovery/TestFSRMStateStore.java | 16 +- .../capacity/TestApplicationLimits.java | 8 +- .../capacity/TestCapacityScheduler.java | 86 +++ .../scheduler/capacity/TestChildQueueOrder.java | 25 +- .../scheduler/capacity/TestLeafQueue.java | 142 ++-- .../scheduler/capacity/TestParentQueue.java | 97 +-- .../scheduler/capacity/TestReservations.java | 147 +++-- .../yarn/server/webproxy/ProxyUriUtils.java | 2 +- .../src/site/markdown/TimelineServer.md | 318 ++++----- .../resources/images/timeline_structure.jpg | Bin 0 -> 23070 bytes 179 files changed, 3007 insertions(+), 1895 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5de4026d/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5de4026d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5de4026d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java ----------------------------------------------------------------------
