Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/adc65104 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/adc65104 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/adc65104 Branch: refs/heads/HDFS-7240 Commit: adc651044c0cacff62a3afa1ef1f833ab47e0413 Parents: 38dddad 56ab02e Author: Anu Engineer <[email protected]> Authored: Tue Apr 4 12:50:22 2017 -0700 Committer: Anu Engineer <[email protected]> Committed: Tue Apr 4 12:50:22 2017 -0700 ---------------------------------------------------------------------- LICENSE.txt | 22 +- .../jdiff/Apache_Hadoop_Common_2.8.0.xml | 37921 +++++++++++++++++ hadoop-common-project/hadoop-common/pom.xml | 15 +- .../src/main/conf/log4j.properties | 10 + .../org/apache/hadoop/conf/Configuration.java | 395 +- .../apache/hadoop/fs/AbstractFileSystem.java | 16 +- .../java/org/apache/hadoop/fs/ChecksumFs.java | 13 +- .../apache/hadoop/fs/DelegateToFileSystem.java | 6 + .../hadoop/fs/FSDataOutputStreamBuilder.java | 142 + .../java/org/apache/hadoop/fs/FileStatus.java | 23 +- .../java/org/apache/hadoop/fs/FileSystem.java | 9 + .../org/apache/hadoop/fs/FilterFileSystem.java | 5 + .../java/org/apache/hadoop/fs/FilterFs.java | 8 +- .../org/apache/hadoop/fs/HarFileSystem.java | 5 + .../java/org/apache/hadoop/fs/ftp/FtpFs.java | 7 + .../org/apache/hadoop/fs/local/RawLocalFs.java | 8 + .../hadoop/fs/permission/FsPermission.java | 7 + .../hadoop/fs/permission/PermissionStatus.java | 13 - .../apache/hadoop/fs/shell/SetReplication.java | 30 +- .../apache/hadoop/fs/shell/XAttrCommands.java | 2 +- .../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 23 +- .../org/apache/hadoop/fs/viewfs/ViewFs.java | 97 +- .../apache/hadoop/ha/ZKFailoverController.java | 3 +- .../hadoop/io/retry/RetryInvocationHandler.java | 17 +- .../main/java/org/apache/hadoop/ipc/Client.java | 4 +- .../apache/hadoop/metrics2/AbstractMetric.java | 3 +- .../org/apache/hadoop/metrics2/MetricsTag.java | 3 +- .../metrics2/impl/AbstractMetricsRecord.java | 3 +- .../org/apache/hadoop/metrics2/impl/MsInfo.java | 4 +- .../hadoop/metrics2/lib/MetricsInfoImpl.java | 3 +- .../hadoop/metrics2/lib/MetricsRegistry.java | 4 +- .../hadoop/metrics2/source/JvmMetricsInfo.java | 6 +- .../hadoop/metrics2/util/MetricsCache.java | 4 +- .../org/apache/hadoop/net/InnerNodeImpl.java | 8 +- .../org/apache/hadoop/net/NetworkTopology.java | 2 +- .../net/NetworkTopologyWithNodeGroup.java | 2 +- .../java/org/apache/hadoop/net/NodeBase.java | 2 +- .../hadoop/net/unix/DomainSocketWatcher.java | 12 +- .../alias/CredentialProviderFactory.java | 16 +- .../java/org/apache/hadoop/util/ConfTest.java | 2 +- .../org/apache/hadoop/util/Progressable.java | 2 +- .../org/apache/hadoop/util/StringUtils.java | 8 +- .../apache/hadoop/util/UTF8ByteArrayUtils.java | 4 +- .../apache/hadoop/util/curator/ChildReaper.java | 4 +- .../src/main/native/gtest/gtest-all.cc | 2 +- .../src/main/native/gtest/include/gtest/gtest.h | 2 +- .../src/main/resources/core-default.xml | 6 + .../src/site/markdown/FileSystemShell.md | 2 +- .../src/site/markdown/filesystem/filesystem.md | 14 +- .../site/markdown/filesystem/introduction.md | 85 + .../markdown/release/2.8.0/CHANGES.2.8.0.md | 2986 ++ .../release/2.8.0/RELEASENOTES.2.8.0.md | 1105 + .../org/apache/hadoop/cli/CLITestHelper.java | 17 +- .../apache/hadoop/cli/util/CommandExecutor.java | 4 +- .../apache/hadoop/cli/util/FSCmdExecutor.java | 4 +- .../apache/hadoop/conf/TestConfiguration.java | 62 +- .../fs/FileContextMainOperationsBaseTest.java | 21 +- .../hadoop/fs/FileSystemContractBaseTest.java | 246 +- .../org/apache/hadoop/fs/TestAfsCheckPath.java | 1 + .../org/apache/hadoop/fs/TestFileStatus.java | 10 +- .../java/org/apache/hadoop/fs/TestFilterFs.java | 12 + .../apache/hadoop/fs/TestLocalFileSystem.java | 54 + .../fs/TestRawLocalFileSystemContract.java | 24 +- .../hadoop/fs/contract/ContractTestUtils.java | 27 + .../hadoop/fs/viewfs/TestViewfsFileStatus.java | 10 +- .../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 141 +- .../org/apache/hadoop/test/MetricsAsserts.java | 9 +- .../hadoop/test/MultithreadedTestUtil.java | 2 +- .../src/test/resources/testConf.xml | 2 +- .../hadoop-kms/src/site/markdown/index.md.vm | 2 +- .../apache/hadoop/oncrpc/SimpleTcpServer.java | 3 +- .../java/org/apache/hadoop/portmap/Portmap.java | 1 + hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 1 - .../main/java/org/apache/hadoop/fs/Hdfs.java | 6 + .../java/org/apache/hadoop/hdfs/DFSClient.java | 10 +- .../hadoop/hdfs/DFSInotifyEventInputStream.java | 4 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 2 +- .../hadoop/hdfs/DFSStripedInputStream.java | 22 +- .../hadoop/hdfs/DFSStripedOutputStream.java | 1 + .../org/apache/hadoop/hdfs/DFSUtilClient.java | 2 +- .../hadoop/hdfs/DistributedFileSystem.java | 81 + .../apache/hadoop/hdfs/FileChecksumHelper.java | 42 +- .../hdfs/client/HdfsClientConfigKeys.java | 2 + .../hadoop/hdfs/protocol/DatanodeInfo.java | 1 + .../hdfs/protocol/FsPermissionExtension.java | 17 +- .../hdfs/server/protocol/SlowDiskReports.java | 146 + .../apache/hadoop/hdfs/web/JsonUtilClient.java | 13 +- .../hadoop/fs/http/client/HttpFSFileSystem.java | 7 +- .../hadoop/fs/http/server/FSOperations.java | 39 +- .../src/site/markdown/index.md | 8 +- .../fs/http/client/BaseTestHttpFSWith.java | 24 +- .../hadoop/fs/http/server/TestHttpFSServer.java | 18 + .../org/apache/hadoop/test/TestHdfsHelper.java | 18 +- .../dev-support/findbugsExcludeFile.xml | 12 + .../jdiff/Apache_Hadoop_HDFS_2.8.0.xml | 312 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 + .../hadoop/hdfs/net/DFSNetworkTopology.java | 349 + .../hadoop/hdfs/net/DFSTopologyNodeImpl.java | 302 + .../DatanodeProtocolClientSideTranslatorPB.java | 7 +- .../DatanodeProtocolServerSideTranslatorPB.java | 3 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 68 + .../server/blockmanagement/BlockManager.java | 12 +- .../blockmanagement/DatanodeDescriptor.java | 9 + .../server/blockmanagement/DatanodeManager.java | 46 +- .../server/blockmanagement/SlowDiskTracker.java | 294 + .../server/blockmanagement/SlowPeerTracker.java | 2 +- .../hdfs/server/datanode/BPServiceActor.java | 40 +- .../server/datanode/BlockChecksumHelper.java | 6 +- .../hdfs/server/datanode/BlockReceiver.java | 61 +- .../server/datanode/BlockRecoveryWorker.java | 10 +- .../hadoop/hdfs/server/datanode/DataNode.java | 87 +- .../hdfs/server/datanode/DataNodeMXBean.java | 7 + .../hdfs/server/datanode/DataStorage.java | 4 +- .../datanode/IncrementalBlockReportManager.java | 41 +- .../server/datanode/checker/AbstractFuture.java | 1295 + .../datanode/checker/DatasetVolumeChecker.java | 15 +- .../checker/StorageLocationChecker.java | 1 + .../datanode/checker/ThrottledAsyncChecker.java | 37 +- .../server/datanode/checker/TimeoutFuture.java | 162 + .../StripedBlockChecksumReconstructor.java | 48 +- .../erasurecode/StripedBlockReconstructor.java | 1 + .../erasurecode/StripedReconstructor.java | 6 + .../datanode/fsdataset/impl/BlockPoolSlice.java | 2 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 14 +- .../datanode/fsdataset/impl/FsVolumeImpl.java | 2 +- .../datanode/fsdataset/impl/FsVolumeList.java | 13 +- .../datanode/metrics/DataNodeDiskMetrics.java | 62 +- .../datanode/metrics/DataNodeMetrics.java | 35 + .../diskbalancer/DiskBalancerException.java | 1 + .../server/diskbalancer/command/Command.java | 29 +- .../diskbalancer/command/ReportCommand.java | 6 +- .../hdfs/server/namenode/AclTransformation.java | 4 +- .../namenode/ErasureCodingPolicyManager.java | 110 +- .../server/namenode/FSDirErasureCodingOp.java | 54 +- .../server/namenode/FSDirStatAndListingOp.java | 10 +- .../server/namenode/FSImageFormatPBINode.java | 3 +- .../hdfs/server/namenode/FSNamesystem.java | 15 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 12 +- .../hadoop/hdfs/server/namenode/JournalSet.java | 2 +- .../hadoop/hdfs/server/namenode/NameNode.java | 6 + .../hdfs/server/namenode/NameNodeRpcServer.java | 6 +- .../server/namenode/NameNodeStatusMXBean.java | 8 + .../ha/ConfiguredFailoverProxyProvider.java | 15 +- .../ha/RequestHedgingProxyProvider.java | 43 +- .../snapshot/DirectoryWithSnapshotFeature.java | 5 + .../hdfs/server/protocol/DatanodeProtocol.java | 3 +- .../apache/hadoop/hdfs/tools/AdminHelper.java | 6 +- .../apache/hadoop/hdfs/tools/CryptoAdmin.java | 2 +- .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 24 +- .../offlineEditsViewer/OfflineEditsViewer.java | 19 +- .../OfflineImageReconstructor.java | 10 +- .../offlineImageViewer/PBImageXmlWriter.java | 21 +- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 6 + .../src/main/proto/DatanodeProtocol.proto | 15 + .../src/main/resources/hdfs-default.xml | 10 + .../src/site/markdown/HDFSCommands.md | 2 +- .../src/site/markdown/HDFSDiskbalancer.md | 4 +- .../src/site/markdown/HDFSErasureCoding.md | 53 +- .../src/site/markdown/HdfsEditsViewer.md | 2 + .../apache/hadoop/cli/TestErasureCodingCLI.java | 3 + .../hadoop/cli/util/CacheAdminCmdExecutor.java | 4 +- .../hadoop/cli/util/CryptoAdminCmdExecutor.java | 4 +- .../cli/util/ErasureCodingCliCmdExecutor.java | 4 +- .../apache/hadoop/hdfs/DFSNetworkTopology.java | 36 - .../org/apache/hadoop/hdfs/DFSTestUtil.java | 35 +- .../apache/hadoop/hdfs/DFSTopologyNodeImpl.java | 253 - .../hdfs/ErasureCodeBenchmarkThroughput.java | 5 +- .../apache/hadoop/hdfs/StripedFileTestUtil.java | 12 + .../hadoop/hdfs/TestDFSNetworkTopology.java | 260 - .../TestDFSRSDefault10x4StripedInputStream.java | 2 +- ...TestDFSRSDefault10x4StripedOutputStream.java | 2 +- ...fault10x4StripedOutputStreamWithFailure.java | 2 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 5 +- .../hadoop/hdfs/TestDFSStripedOutputStream.java | 4 +- .../TestDFSStripedOutputStreamWithFailure.java | 4 +- .../hdfs/TestDFSXORStripedInputStream.java | 2 +- .../hdfs/TestDFSXORStripedOutputStream.java | 2 +- ...estDFSXORStripedOutputStreamWithFailure.java | 2 +- .../hadoop/hdfs/TestDataTransferProtocol.java | 2 +- .../apache/hadoop/hdfs/TestDecommission.java | 98 + .../hdfs/TestDecommissionWithStriped.java | 7 +- .../hadoop/hdfs/TestDistributedFileSystem.java | 57 +- .../TestErasureCodeBenchmarkThroughput.java | 2 + .../hadoop/hdfs/TestErasureCodingPolicies.java | 49 +- .../TestErasureCodingPolicyWithSnapshot.java | 31 +- .../apache/hadoop/hdfs/TestFileChecksum.java | 7 +- .../org/apache/hadoop/hdfs/TestFileStatus.java | 18 +- .../hadoop/hdfs/TestFileStatusWithECPolicy.java | 28 +- .../hadoop/hdfs/TestLeaseRecoveryStriped.java | 5 +- .../hadoop/hdfs/TestMaintenanceState.java | 51 + .../hdfs/TestReadStripedFileWithDecoding.java | 7 +- .../TestReadStripedFileWithMissingBlocks.java | 5 +- .../hadoop/hdfs/TestReconstructStripedFile.java | 9 +- .../org/apache/hadoop/hdfs/TestSafeMode.java | 24 + .../hdfs/TestSafeModeWithStripedFile.java | 7 +- .../hadoop/hdfs/TestSetrepIncreasing.java | 44 + .../TestUnsetAndChangeDirectoryEcPolicy.java | 8 +- .../hadoop/hdfs/TestWriteReadStripedFile.java | 7 +- .../hdfs/TestWriteStripedFileWithFailure.java | 5 +- .../hadoop/hdfs/net/TestDFSNetworkTopology.java | 568 + .../hadoop/hdfs/protocolPB/TestPBHelper.java | 39 +- .../hdfs/qjournal/client/TestQJMWithFaults.java | 2 +- .../client/TestQuorumJournalManager.java | 2 +- .../hdfs/server/balancer/TestBalancer.java | 18 +- .../blockmanagement/TestBlockInfoStriped.java | 4 +- .../blockmanagement/TestBlockManager.java | 113 +- .../TestBlockTokenWithDFSStriped.java | 10 +- .../TestLowRedundancyBlockQueues.java | 4 +- .../TestNameNodePrunesMissingStorages.java | 4 +- ...constructStripedBlocksWithRackAwareness.java | 16 +- .../TestSequentialBlockGroupId.java | 8 +- .../blockmanagement/TestSlowDiskTracker.java | 439 + .../TestSortLocatedStripedBlock.java | 4 +- .../datanode/InternalDataNodeTestUtils.java | 4 +- .../server/datanode/TestBPOfferService.java | 4 +- .../datanode/TestBlockCountersInPendingIBR.java | 146 + .../hdfs/server/datanode/TestBlockRecovery.java | 7 +- .../datanode/TestBpServiceActorScheduler.java | 20 +- .../TestDataNodeErasureCodingMetrics.java | 7 +- .../server/datanode/TestDataNodeLifeline.java | 8 +- .../server/datanode/TestDataNodeMXBean.java | 33 +- .../TestDataNodeVolumeFailureReporting.java | 116 +- .../hdfs/server/datanode/TestDataStorage.java | 2 +- .../TestDatanodeProtocolRetryPolicy.java | 4 +- .../server/datanode/TestFsDatasetCache.java | 4 +- .../hdfs/server/datanode/TestStorageReport.java | 4 +- .../TestDatasetVolumeCheckerTimeout.java | 134 + .../checker/TestThrottledAsyncChecker.java | 14 +- .../TestThrottledAsyncCheckerTimeout.java | 223 + .../command/TestDiskBalancerCommand.java | 71 +- .../hadoop/hdfs/server/mover/TestMover.java | 7 +- .../hdfs/server/namenode/FSAclBaseTest.java | 25 +- .../server/namenode/NNThroughputBenchmark.java | 7 +- .../hdfs/server/namenode/NameNodeAdapter.java | 3 +- .../TestAddOverReplicatedStripedBlocks.java | 8 +- .../namenode/TestAddStripedBlockInFBR.java | 8 +- .../server/namenode/TestAddStripedBlocks.java | 13 +- .../hdfs/server/namenode/TestDeadDatanode.java | 4 +- .../server/namenode/TestEnabledECPolicies.java | 162 + .../server/namenode/TestFSEditLogLoader.java | 7 +- .../hdfs/server/namenode/TestFSImage.java | 10 +- .../namenode/TestFavoredNodesEndToEnd.java | 23 + .../hadoop/hdfs/server/namenode/TestFsck.java | 41 +- .../server/namenode/TestNameNodeMXBean.java | 14 +- .../namenode/TestNameNodeStatusMXBean.java | 148 + .../namenode/TestQuotaWithStripedBlocks.java | 5 +- .../namenode/TestReconstructStripedBlocks.java | 12 +- .../server/namenode/TestStripedINodeFile.java | 9 +- .../ha/TestRequestHedgingProxyProvider.java | 108 +- .../namenode/ha/TestRetryCacheWithHA.java | 2 +- .../snapshot/TestRenameWithSnapshots.java | 6 +- .../namenode/snapshot/TestSnapshotDeletion.java | 75 + .../shortcircuit/TestShortCircuitLocalRead.java | 2 +- .../TestOfflineEditsViewer.java | 18 + .../TestOfflineImageViewer.java | 108 +- ...TestOfflineImageViewerWithStripedBlocks.java | 10 +- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 72 + .../src/test/resources/testCryptoConf.xml | 31 + .../test/resources/testErasureCodingConf.xml | 37 +- .../Apache_Hadoop_MapReduce_Common_2.8.0.xml | 113 + .../Apache_Hadoop_MapReduce_Core_2.8.0.xml | 27490 ++++++++++++ .../Apache_Hadoop_MapReduce_JobClient_2.8.0.xml | 16 + .../v2/app/job/impl/TaskAttemptImpl.java | 285 +- .../mapreduce/v2/app/webapp/ConfBlock.java | 2 +- .../v2/app/rm/TestRMContainerAllocator.java | 453 +- .../hadoop/filecache/DistributedCache.java | 8 +- .../java/org/apache/hadoop/mapred/JobConf.java | 8 +- .../mapred/lib/FieldSelectionMapReduce.java | 2 +- .../mapred/lib/KeyFieldBasedComparator.java | 2 +- .../hadoop/mapreduce/JobResourceUploader.java | 36 +- .../apache/hadoop/mapreduce/JobSubmitter.java | 2 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 8 +- .../mapreduce/filecache/DistributedCache.java | 8 +- .../lib/fieldsel/FieldSelectionHelper.java | 7 +- .../lib/fieldsel/FieldSelectionMapper.java | 2 +- .../lib/fieldsel/FieldSelectionReducer.java | 2 +- .../lib/input/KeyValueLineRecordReader.java | 13 +- .../mapreduce/lib/jobcontrol/JobControl.java | 2 +- .../mapreduce/lib/output/TextOutputFormat.java | 9 +- .../lib/partition/KeyFieldBasedComparator.java | 4 +- .../lib/partition/KeyFieldBasedPartitioner.java | 2 +- .../hadoop/mapreduce/util/ConfigUtil.java | 8 +- .../mapreduce/TestJobResourceUploader.java | 40 +- .../org/apache/hadoop/mapred/YARNRunner.java | 4 +- .../apache/hadoop/fs/slive/OperationOutput.java | 4 +- .../hadoop/mapred/TestFieldSelection.java | 2 +- .../apache/hadoop/mapred/TestYARNRunner.java | 2 +- .../mapred/lib/TestKeyFieldBasedComparator.java | 4 +- .../lib/fieldsel/TestMRFieldSelection.java | 2 +- .../TestMRKeyFieldBasedComparator.java | 4 +- .../apache/hadoop/mapred/ShuffleHandler.java | 68 +- .../hadoop/mapred/TestShuffleHandler.java | 31 + hadoop-mapreduce-project/pom.xml | 1 + hadoop-project-dist/pom.xml | 2 +- hadoop-project/pom.xml | 22 +- .../oss/TestAliyunOSSFileSystemContract.java | 35 +- .../hadoop/fs/s3a/S3ABlockOutputStream.java | 5 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 10 +- .../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 34 +- .../apache/hadoop/fs/s3a/S3ClientFactory.java | 7 +- .../fs/s3a/SimpleAWSCredentialsProvider.java | 3 +- .../fs/s3a/TemporaryAWSCredentialsProvider.java | 3 +- .../src/site/markdown/tools/hadoop-aws/index.md | 3 + .../site/markdown/tools/hadoop-aws/testing.md | 1 + .../fs/s3a/ITestS3AAWSCredentialsProvider.java | 44 +- .../fs/s3a/ITestS3AFileSystemContract.java | 39 +- .../fs/s3a/ITestS3ATemporaryCredentials.java | 2 +- .../hadoop/fs/s3a/MockS3ClientFactory.java | 2 +- .../fs/s3a/TestS3AAWSCredentialsProvider.java | 16 +- .../hadoop/fs/s3a/TestS3AGetFileStatus.java | 9 + .../hadoop/fs/s3native/TestS3Credentials.java | 18 - .../org/apache/hadoop/fs/adl/AdlConfKeys.java | 39 +- .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 4 + .../META-INF/org.apache.hadoop.fs.FileSystem | 16 - .../src/site/markdown/index.md | 87 +- .../hadoop/fs/adl/TestAzureADTokenProvider.java | 5 +- .../apache/hadoop/fs/adl/TestGetFileStatus.java | 13 +- .../fs/adl/TestValidateConfiguration.java | 83 +- .../adl/live/TestAdlFileSystemContractLive.java | 3 +- .../fs/azure/AzureNativeFileSystemStore.java | 2 +- .../hadoop/fs/azure/BlockBlobAppendStream.java | 2 +- .../hadoop/fs/azure/NativeAzureFileSystem.java | 69 +- .../fs/azure/RemoteSASKeyGeneratorImpl.java | 191 +- .../fs/azure/RemoteWasbAuthorizerImpl.java | 107 +- .../fs/azure/SecureStorageInterfaceImpl.java | 11 +- .../fs/azure/WasbAuthorizerInterface.java | 5 +- .../hadoop/fs/azure/WasbRemoteCallHelper.java | 3 +- .../hadoop/fs/azure/security/Constants.java | 54 + .../hadoop/fs/azure/security/SecurityUtils.java | 86 + .../security/WasbDelegationTokenIdentifier.java | 48 + .../fs/azure/security/WasbTokenRenewer.java | 150 + .../hadoop/fs/azure/security/package-info.java | 22 + ...apache.hadoop.security.token.TokenIdentifier | 16 + ...rg.apache.hadoop.security.token.TokenRenewer | 16 + .../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 4 +- .../org/apache/hadoop/tools/CopyListing.java | 73 +- .../hadoop/tools/CopyListingFileStatus.java | 91 +- .../java/org/apache/hadoop/tools/DistCp.java | 96 +- .../org/apache/hadoop/tools/DistCpContext.java | 198 + .../apache/hadoop/tools/DistCpOptionSwitch.java | 12 +- .../org/apache/hadoop/tools/DistCpOptions.java | 913 +- .../org/apache/hadoop/tools/DistCpSync.java | 42 +- .../hadoop/tools/FileBasedCopyListing.java | 12 +- .../apache/hadoop/tools/GlobbedCopyListing.java | 17 +- .../org/apache/hadoop/tools/OptionsParser.java | 271 +- .../apache/hadoop/tools/SimpleCopyListing.java | 190 +- .../hadoop/tools/mapred/CopyCommitter.java | 189 +- .../apache/hadoop/tools/mapred/CopyMapper.java | 40 +- .../tools/mapred/RetriableFileCopyCommand.java | 29 +- .../tools/mapred/UniformSizeInputFormat.java | 5 +- .../apache/hadoop/tools/util/DistCpUtils.java | 126 +- .../src/site/markdown/DistCp.md.vm | 3 +- .../apache/hadoop/tools/TestCopyListing.java | 51 +- .../hadoop/tools/TestCopyListingFileStatus.java | 1 + .../apache/hadoop/tools/TestDistCpOptions.java | 500 + .../org/apache/hadoop/tools/TestDistCpSync.java | 68 +- .../hadoop/tools/TestDistCpSyncReverseBase.java | 44 +- .../apache/hadoop/tools/TestDistCpSystem.java | 368 +- .../apache/hadoop/tools/TestDistCpViewFs.java | 10 +- .../hadoop/tools/TestFileBasedCopyListing.java | 9 +- .../hadoop/tools/TestGlobbedCopyListing.java | 11 +- .../apache/hadoop/tools/TestIntegration.java | 20 +- .../apache/hadoop/tools/TestOptionsParser.java | 81 +- .../contract/AbstractContractDistCpTest.java | 6 +- .../hadoop/tools/mapred/TestCopyCommitter.java | 39 +- .../mapred/TestUniformSizeInputFormat.java | 15 +- .../mapred/lib/TestDynamicInputFormat.java | 17 +- .../hadoop/tools/util/TestDistCpUtils.java | 123 +- .../services/org.apache.hadoop.fs.FileSystem | 16 - .../org/apache/hadoop/yarn/sls/SLSRunner.java | 22 +- .../hadoop/yarn/sls/appmaster/AMSimulator.java | 29 +- .../sls/scheduler/FairSchedulerMetrics.java | 308 +- .../sls/scheduler/ResourceSchedulerWrapper.java | 972 - .../sls/scheduler/SLSCapacityScheduler.java | 686 +- .../yarn/sls/scheduler/SLSFairScheduler.java | 339 + .../yarn/sls/scheduler/SchedulerMetrics.java | 602 +- .../yarn/sls/scheduler/SchedulerWrapper.java | 23 +- .../hadoop/yarn/sls/scheduler/Tracker.java | 46 + .../apache/hadoop/yarn/sls/utils/SLSUtils.java | 1 + .../apache/hadoop/yarn/sls/web/SLSWebApp.java | 12 +- .../yarn/sls/appmaster/TestAMSimulator.java | 87 +- .../yarn/sls/nodemanager/TestNMSimulator.java | 32 +- .../hadoop/streaming/StreamKeyValUtil.java | 8 +- .../jdiff/Apache_Hadoop_YARN_Client_2.8.0.xml | 2316 + .../jdiff/Apache_Hadoop_YARN_Common_2.8.0.xml | 2665 ++ .../Apache_Hadoop_YARN_Server_Common_2.8.0.xml | 829 + .../records/ApplicationSubmissionContext.java | 57 +- .../org/apache/hadoop/yarn/conf/HAUtil.java | 30 +- .../hadoop/yarn/conf/YarnConfiguration.java | 28 +- .../src/main/proto/yarn_protos.proto | 2 +- .../distributedshell/TestDistributedShell.java | 30 +- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 178 +- .../hadoop/yarn/client/TestRMFailover.java | 39 +- .../yarn/client/api/impl/TestAMRMClient.java | 272 +- .../api/impl/TestAMRMClientOnRMRestart.java | 59 +- .../hadoop/yarn/client/cli/TestLogsCLI.java | 31 + .../pb/ApplicationSubmissionContextPBImpl.java | 66 +- .../yarn/api/records/impl/pb/ProtoUtils.java | 6 +- .../yarn/client/api/impl/TimelineConnector.java | 4 +- .../client/api/impl/TimelineV2ClientImpl.java | 9 +- .../logaggregation/AggregatedLogFormat.java | 52 +- .../yarn/logaggregation/LogCLIHelpers.java | 11 +- .../nodelabels/CommonNodeLabelsManager.java | 22 + .../hadoop/yarn/util/resource/Resources.java | 7 + .../org/apache/hadoop/yarn/webapp/WebApp.java | 4 +- .../src/main/resources/yarn-default.xml | 20 + .../org/apache/hadoop/yarn/conf/TestHAUtil.java | 86 +- .../yarn/util/resource/TestResources.java | 28 +- .../client/impl/zk/RegistrySecurity.java | 12 +- .../webapp/AHSWebServices.java | 32 +- .../timeline/RollingLevelDBTimelineStore.java | 263 +- .../webapp/TestAHSWebServices.java | 17 + .../webapp/TestTimelineWebServices.java | 27 - .../server/webapp/YarnWebServiceParams.java | 1 + .../nodemanager/NodeStatusUpdaterImpl.java | 4 +- .../containermanager/ContainerManagerImpl.java | 57 +- .../application/ApplicationImpl.java | 4 +- .../containermanager/container/Container.java | 2 + .../container/ContainerImpl.java | 8 + .../linux/runtime/docker/DockerPullCommand.java | 31 + .../logaggregation/AppLogAggregatorImpl.java | 2 - .../timelineservice/NMTimelinePublisher.java | 8 + .../nodemanager/webapp/NMWebServices.java | 6 +- .../nodemanager/TestNodeStatusUpdater.java | 2 +- .../TestContainerManagerRecovery.java | 18 +- .../launcher/TestContainerLaunch.java | 12 +- .../runtime/docker/TestDockerPullCommand.java | 49 + .../TestLocalCacheDirectoryManager.java | 6 +- .../TestAppLogAggregatorImpl.java | 8 +- .../TestNMTimelinePublisher.java | 10 +- .../nodemanager/webapp/MockContainer.java | 5 + .../nodemanager/webapp/TestNMWebServices.java | 6 + .../server/resourcemanager/ClientRMService.java | 290 +- .../server/resourcemanager/RMAppManager.java | 110 +- .../server/resourcemanager/RMAuditLogger.java | 5 + .../server/resourcemanager/RMServerUtils.java | 68 +- .../server/resourcemanager/ResourceManager.java | 2 +- .../capacity/IntraQueueCandidatesSelector.java | 5 + .../recovery/LeveldbRMStateStore.java | 8 + .../server/resourcemanager/rmapp/RMApp.java | 3 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 54 +- .../rmapp/attempt/RMAppAttemptImpl.java | 35 +- .../scheduler/AbstractYarnScheduler.java | 5 + .../scheduler/ClusterNodeTracker.java | 65 +- .../resourcemanager/scheduler/QueueMetrics.java | 13 + .../scheduler/ResourceScheduler.java | 9 + .../scheduler/capacity/CapacityScheduler.java | 2 +- .../capacity/CapacitySchedulerQueueManager.java | 7 +- .../scheduler/capacity/ParentQueue.java | 6 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 10 +- .../fair/AllocationFileLoaderService.java | 2 +- .../scheduler/fair/FSAppAttempt.java | 54 +- .../scheduler/fair/FSLeafQueue.java | 24 +- .../scheduler/fair/FSParentQueue.java | 21 + .../resourcemanager/scheduler/fair/FSQueue.java | 41 +- .../scheduler/fair/FairScheduler.java | 28 +- .../scheduler/fair/QueueManager.java | 2 +- .../scheduler/policy/OrderingPolicy.java | 105 +- .../resourcemanager/webapp/dao/AppInfo.java | 2 +- .../server/resourcemanager/ACLsTestBase.java | 10 - .../server/resourcemanager/Application.java | 2 +- .../yarn/server/resourcemanager/MockRM.java | 45 +- .../server/resourcemanager/RMHATestBase.java | 20 +- .../ReservationACLsTestBase.java | 5 +- .../server/resourcemanager/TestAppManager.java | 195 +- .../resourcemanager/TestApplicationCleanup.java | 44 +- .../TestApplicationMasterLauncher.java | 11 +- .../TestApplicationMasterService.java | 19 +- .../resourcemanager/TestClientRMService.java | 8 +- .../TestNodeBlacklistingOnAMFailures.java | 213 +- .../yarn/server/resourcemanager/TestRM.java | 10 +- .../resourcemanager/TestRMAdminService.java | 95 + .../resourcemanager/TestRMServerUtils.java | 297 + .../TestReservationSystemWithRMHA.java | 5 +- .../applicationsmanager/MockAsm.java | 6 +- .../TestAMRMRPCNodeUpdates.java | 18 +- .../TestRMAppLogAggregationStatus.java | 2 + .../metrics/TestSystemMetricsPublisher.java | 3 +- ...ionalCapacityPreemptionPolicyIntraQueue.java | 55 + .../recovery/TestLeveldbRMStateStore.java | 19 +- .../resourcemanager/resource/TestResources.java | 4 +- .../resourcetracker/TestNMReconnect.java | 14 +- .../server/resourcemanager/rmapp/MockRMApp.java | 11 +- .../rmapp/TestRMAppTransitions.java | 9 +- .../attempt/TestRMAppAttemptTransitions.java | 19 +- .../rmcontainer/TestRMContainerImpl.java | 1 - .../capacity/TestApplicationLimits.java | 4 +- .../TestApplicationLimitsByPartition.java | 4 +- .../capacity/TestApplicationPriority.java | 29 +- .../capacity/TestCapacityScheduler.java | 2 +- .../scheduler/capacity/TestLeafQueue.java | 3 +- .../scheduler/capacity/TestParentQueue.java | 18 +- .../scheduler/fair/TestFairScheduler.java | 182 +- .../fair/TestFairSchedulerPreemption.java | 12 +- .../security/TestClientToAMTokens.java | 23 +- .../webapp/TestRMWebServicesApps.java | 10 +- .../pom.xml | 19 + .../collector/TimelineCollector.java | 37 +- .../collector/TimelineCollectorManager.java | 7 +- .../collector/TimelineCollectorWebService.java | 12 +- .../reader/TimelineReaderServer.java | 9 + .../collector/TestTimelineCollector.java | 63 + .../src/site/markdown/FairScheduler.md | 40 +- .../src/site/markdown/TimelineServiceV2.md | 9 + .../hadoop-yarn/hadoop-yarn-ui/pom.xml | 20 +- .../app/components/base-chart-component.js | 4 + .../main/webapp/app/components/nodes-heatmap.js | 106 +- .../main/webapp/app/helpers/log-files-comma.js | 10 +- .../app/serializers/yarn-node-container.js | 2 +- .../main/webapp/app/serializers/yarn-node.js | 4 +- .../main/webapp/app/serializers/yarn-rm-node.js | 4 +- .../src/main/webapp/app/styles/app.css | 12 + .../src/main/webapp/app/templates/yarn-node.hbs | 23 +- .../main/webapp/app/templates/yarn-nodes.hbs | 2 +- .../webapp/app/templates/yarn-nodes/table.hbs | 13 +- .../hadoop-yarn-ui/src/main/webapp/package.json | 6 +- 516 files changed, 92672 insertions(+), 6565 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties ---------------------------------------------------------------------- diff --cc hadoop-common-project/hadoop-common/src/main/conf/log4j.properties index 9984666,b4658ae..1289115 --- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties +++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties @@@ -322,26 -322,12 +322,36 @@@ log4j.appender.EWMA.cleanupInterval=${y log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds} log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages} + # Fair scheduler requests log on state dump + log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=DEBUG,FSLOGGER + log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=false + log4j.appender.FSLOGGER=org.apache.log4j.RollingFileAppender + log4j.appender.FSLOGGER.File=${hadoop.log.dir}/fairscheduler-statedump.log + log4j.appender.FSLOGGER.layout=org.apache.log4j.PatternLayout + log4j.appender.FSLOGGER.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n + log4j.appender.FSLOGGER.MaxFileSize=${hadoop.log.maxfilesize} -log4j.appender.FSLOGGER.MaxBackupIndex=${hadoop.log.maxbackupindex} ++log4j.appender.FSLOGGER.MaxBackupIndex=${hadoop.log.maxbackupindex} ++ +# +# Add a logger for ozone that is separate from the Datanode. +# +log4j.logger.org.apache.hadoop.ozone=DEBUG,OZONE,FILE + +# Do not log into datanode logs. Remove this line to have single log. +log4j.additivity.org.apache.hadoop.ozone=false + +# For development purposes, log both to console and log file. +log4j.appender.OZONE=org.apache.log4j.ConsoleAppender +log4j.appender.OZONE.Threshold=info +log4j.appender.OZONE.layout=org.apache.log4j.PatternLayout +log4j.appender.OZONE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p \ + %X{component} %X{function} %X{resource} %X{user} %X{request} - %m%n + +# Real ozone logger that writes to ozone.log +log4j.appender.FILE=org.apache.log4j.DailyRollingFileAppender +log4j.appender.FILE.File=${hadoop.log.dir}/ozone.log +log4j.appender.FILE.Threshold=debug +log4j.appender.FILE.layout=org.apache.log4j.PatternLayout +log4j.appender.FILE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p \ +(%F:%L) %X{function} %X{resource} %X{user} %X{request} - \ +%m%n http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java ---------------------------------------------------------------------- diff --cc hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java index f7ab52e,bd48b15..4eda7f2 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java @@@ -91,7 -92,7 +92,7 @@@ public class SimpleTcpServer LOG.info("Started listening to TCP requests at port " + boundPort + " for " + rpcProgram + " with workerCount " + workerCount); } -- ++ // boundPort will be set only after server starts public int getBoundPort() { return this.boundPort; http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-project/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java index 5f2265b,a2105c7..e22a3a2 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java @@@ -82,14 -103,24 +103,24 @@@ public class RemoteWasbAuthorizerImpl i @Override public void init(Configuration conf) throws WasbAuthorizationException, IOException { + LOG.debug("Initializing RemoteWasbAuthorizerImpl instance"); + Iterator<Token<? extends TokenIdentifier>> tokenIterator = null; + try { - delegationToken = SecurityUtils.getDelegationTokenFromCredentials(); ++ delegationToken = SecurityUtils.getDelegationTokenFromCredentials(); + } catch (IOException e) { + final String msg = "Error in fetching the WASB delegation token"; + LOG.error(msg, e); + throw new IOException(msg, e); + } - remoteAuthorizerServiceUrl = conf.get(KEY_REMOTE_AUTH_SERVICE_URL); + remoteAuthorizerServiceUrl = SecurityUtils + .getRemoteAuthServiceUrls(conf); if (remoteAuthorizerServiceUrl == null -- || remoteAuthorizerServiceUrl.isEmpty()) { ++ || remoteAuthorizerServiceUrl.isEmpty()) { throw new WasbAuthorizationException( "fs.azure.authorization.remote.service.url config not set" - + " in configuration."); + + " in configuration."); } this.remoteCallHelper = new WasbRemoteCallHelper(); @@@ -96,41 -130,86 +130,86 @@@ } @Override - public boolean authorize(String wasbAbsolutePath, String accessType, - String delegationToken) throws WasbAuthorizationException, IOException { - + public boolean authorize(String wasbAbsolutePath, String accessType) + throws WasbAuthorizationException, IOException { + try { + URIBuilder uriBuilder = new URIBuilder(remoteAuthorizerServiceUrl); + uriBuilder.setPath("/" + CHECK_AUTHORIZATION_OP); + uriBuilder.addParameter(WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME, + wasbAbsolutePath); + uriBuilder.addParameter(ACCESS_OPERATION_QUERY_PARAM_NAME, + accessType); - uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, - delegationToken); ++ if (isSecurityEnabled && StringUtils.isNotEmpty(delegationToken)) { ++ uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, ++ delegationToken); ++ } + - String responseBody = remoteCallHelper.makeRemoteGetRequest( - new HttpGet(uriBuilder.build())); ++ String responseBody = null; ++ UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); ++ UserGroupInformation connectUgi = ugi.getRealUser(); ++ if (connectUgi == null) { ++ connectUgi = ugi; ++ } else { ++ uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName()); ++ } ++ if (isSecurityEnabled && !connectUgi.hasKerberosCredentials()) { ++ connectUgi = UserGroupInformation.getLoginUser(); ++ } ++ connectUgi.checkTGTAndReloginFromKeytab(); ++ + try { - URIBuilder uriBuilder = new URIBuilder(remoteAuthorizerServiceUrl); - uriBuilder.setPath("/" + CHECK_AUTHORIZATION_OP); - uriBuilder.addParameter(WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME, - wasbAbsolutePath); - uriBuilder.addParameter(ACCESS_OPERATION_QUERY_PARAM_NAME, - accessType); - if (isSecurityEnabled && StringUtils.isNotEmpty(delegationToken)) { - uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, - delegationToken); - } - - String responseBody = null; - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - UserGroupInformation connectUgi = ugi.getRealUser(); - if (connectUgi == null) { - connectUgi = ugi; - } else { - uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName()); - } - if (isSecurityEnabled && !connectUgi.hasKerberosCredentials()) { - connectUgi = UserGroupInformation.getLoginUser(); - } - connectUgi.checkTGTAndReloginFromKeytab(); - - try { - responseBody = connectUgi - .doAs(new PrivilegedExceptionAction<String>() { - @Override - public String run() throws Exception { - AuthenticatedURL.Token token = null; - HttpGet httpGet = new HttpGet(uriBuilder.build()); - if (isKerberosSupportEnabled && UserGroupInformation - .isSecurityEnabled() && (delegationToken == null - || delegationToken.isEmpty())) { - token = new AuthenticatedURL.Token(); - final Authenticator kerberosAuthenticator = new KerberosDelegationTokenAuthenticator(); - try { - kerberosAuthenticator - .authenticate(uriBuilder.build().toURL(), token); - Validate.isTrue(token.isSet(), - "Authenticated Token is NOT present. The request cannot proceed."); - } catch (AuthenticationException e){ - throw new IOException("Authentication failed in check authorization", e); - } - if (token != null) { - httpGet.setHeader("Cookie", - AuthenticatedURL.AUTH_COOKIE + "=" + token); - } ++ responseBody = connectUgi ++ .doAs(new PrivilegedExceptionAction<String>() { ++ @Override ++ public String run() throws Exception { ++ AuthenticatedURL.Token token = null; ++ HttpGet httpGet = new HttpGet(uriBuilder.build()); ++ if (isKerberosSupportEnabled && UserGroupInformation ++ .isSecurityEnabled() && (delegationToken == null ++ || delegationToken.isEmpty())) { ++ token = new AuthenticatedURL.Token(); ++ final Authenticator kerberosAuthenticator = new KerberosDelegationTokenAuthenticator(); ++ try { ++ kerberosAuthenticator ++ .authenticate(uriBuilder.build().toURL(), token); ++ Validate.isTrue(token.isSet(), ++ "Authenticated Token is NOT present. The request cannot proceed."); ++ } catch (AuthenticationException e){ ++ throw new IOException("Authentication failed in check authorization", e); ++ } ++ if (token != null) { ++ httpGet.setHeader("Cookie", ++ AuthenticatedURL.AUTH_COOKIE + "=" + token); + } - return remoteCallHelper.makeRemoteGetRequest(httpGet); + } - }); - } catch (InterruptedException e) { - LOG.error("Error in check authorization", e); - throw new WasbAuthorizationException("Error in check authorize", e); - } - - ObjectMapper objectMapper = new ObjectMapper(); - RemoteAuthorizerResponse authorizerResponse = - objectMapper - .readValue(responseBody, RemoteAuthorizerResponse.class); - - if (authorizerResponse == null) { - throw new WasbAuthorizationException( - "RemoteAuthorizerResponse object null from remote call"); - } else if (authorizerResponse.getResponseCode() - == REMOTE_CALL_SUCCESS_CODE) { - return authorizerResponse.getAuthorizationResult(); - } else { - throw new WasbAuthorizationException("Remote authorization" - + " serivce encountered an error " - + authorizerResponse.getResponseMessage()); - } - } catch (URISyntaxException | WasbRemoteCallException - | JsonParseException | JsonMappingException ex) { - throw new WasbAuthorizationException(ex); ++ return remoteCallHelper.makeRemoteGetRequest(httpGet); ++ } ++ }); ++ } catch (InterruptedException e) { ++ LOG.error("Error in check authorization", e); ++ throw new WasbAuthorizationException("Error in check authorize", e); + } + + ObjectMapper objectMapper = new ObjectMapper(); + RemoteAuthorizerResponse authorizerResponse = - objectMapper.readValue(responseBody, RemoteAuthorizerResponse.class); ++ objectMapper ++ .readValue(responseBody, RemoteAuthorizerResponse.class); + + if (authorizerResponse == null) { + throw new WasbAuthorizationException( + "RemoteAuthorizerResponse object null from remote call"); + } else if (authorizerResponse.getResponseCode() + == REMOTE_CALL_SUCCESS_CODE) { + return authorizerResponse.getAuthorizationResult(); + } else { + throw new WasbAuthorizationException("Remote authorization" - + " service encountered an error " ++ + " serivce encountered an error " + + authorizerResponse.getResponseMessage()); + } + } catch (URISyntaxException | WasbRemoteCallException + | JsonParseException | JsonMappingException ex) { + throw new WasbAuthorizationException(ex); + } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java ---------------------------------------------------------------------- diff --cc hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java index 43c0271,06ba137..6269f21 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java @@@ -99,7 -100,7 +100,7 @@@ public class TestAMRMClient private List<NodeReport> nodeReports = null; private ApplicationAttemptId attemptId = null; private int nodeCount = 3; -- ++ static final int rolling_interval_sec = 13; static final long am_expire_ms = 4000; @@@ -126,12 -127,16 +127,16 @@@ @Before public void setup() throws Exception { - // start minicluster conf = new YarnConfiguration(); + createClusterAndStartApplication(); + } + + private void createClusterAndStartApplication() throws Exception { + // start minicluster conf.set(YarnConfiguration.RM_SCHEDULER, schedulerName); conf.setLong( -- YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, -- rolling_interval_sec); ++ YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, ++ rolling_interval_sec); conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms); conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100); // set the minimum allocation so that resource decrease can go under 1024 @@@ -148,7 -153,7 +153,7 @@@ // get node info assertTrue("All node managers did not connect to the RM within the " -- + "allotted 5-second timeout", ++ + "allotted 5-second timeout", yarnCluster.waitForNodeManagersToConnect(5000L)); nodeReports = yarnClient.getNodeReports(NodeState.RUNNING); assertEquals("Not all node managers were reported running", @@@ -164,7 -169,7 +169,7 @@@ racks = new String[]{ rack }; // submit new app -- ApplicationSubmissionContext appContext = ++ ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); // set the application name @@@ -178,10 -183,10 +183,10 @@@ // Set up the container launch context for the application master ContainerLaunchContext amContainer = BuilderUtils.newContainerLaunchContext( -- Collections.<String, LocalResource> emptyMap(), -- new HashMap<String, String>(), Arrays.asList("sleep", "100"), -- new HashMap<String, ByteBuffer>(), null, -- new HashMap<ApplicationAccessType, String>()); ++ Collections.<String, LocalResource> emptyMap(), ++ new HashMap<String, String>(), Arrays.asList("sleep", "100"), ++ new HashMap<String, ByteBuffer>(), null, ++ new HashMap<ApplicationAccessType, String>()); appContext.setAMContainerSpec(amContainer); appContext.setResource(Resource.newInstance(1024, 1)); // Create the request to send to the applications manager @@@ -199,7 -204,7 +204,7 @@@ attemptId = appReport.getCurrentApplicationAttemptId(); appAttempt = yarnCluster.getResourceManager().getRMContext().getRMApps() -- .get(attemptId.getApplicationId()).getCurrentAppAttempt(); ++ .get(attemptId.getApplicationId()).getCurrentAppAttempt(); while (true) { if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) { break; @@@ -211,14 -216,14 +216,14 @@@ // Just dig into the ResourceManager and get the AMRMToken just for the sake // of testing. UserGroupInformation.setLoginUser(UserGroupInformation -- .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName())); ++ .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName())); // emulate RM setup of AMRM token in credentials by adding the token // *before* setting the token service UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken()); appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf)); } -- ++ @After public void teardown() throws YarnException, IOException { yarnClient.killApplication(attemptId.getApplicationId()); @@@ -245,7 -250,7 +250,7 @@@ amClient.getMatchingRequests(priority, node, testCapability1); assertEquals("Expected no matching requests.", matches.size(), 0); } -- ++ @Test (timeout=60000) public void testAMRMClientMatchingFit() throws YarnException, IOException { AMRMClient<ContainerRequest> amClient = null; @@@ -255,7 -260,7 +260,7 @@@ amClient.init(conf); amClient.start(); amClient.registerApplicationMaster("Host", 10000, ""); -- ++ Resource capability1 = Resource.newInstance(1024, 2); Resource capability2 = Resource.newInstance(1024, 1); Resource capability3 = Resource.newInstance(1000, 2); @@@ -264,19 -269,19 +269,19 @@@ Resource capability6 = Resource.newInstance(2000, 1); Resource capability7 = Resource.newInstance(2000, 1); -- ContainerRequest storedContainer1 = ++ ContainerRequest storedContainer1 = new ContainerRequest(capability1, nodes, racks, priority); -- ContainerRequest storedContainer2 = ++ ContainerRequest storedContainer2 = new ContainerRequest(capability2, nodes, racks, priority); -- ContainerRequest storedContainer3 = ++ ContainerRequest storedContainer3 = new ContainerRequest(capability3, nodes, racks, priority); -- ContainerRequest storedContainer4 = ++ ContainerRequest storedContainer4 = new ContainerRequest(capability4, nodes, racks, priority); -- ContainerRequest storedContainer5 = ++ ContainerRequest storedContainer5 = new ContainerRequest(capability5, nodes, racks, priority); -- ContainerRequest storedContainer6 = ++ ContainerRequest storedContainer6 = new ContainerRequest(capability6, nodes, racks, priority); -- ContainerRequest storedContainer7 = ++ ContainerRequest storedContainer7 = new ContainerRequest(capability7, nodes, racks, priority2, false); amClient.addContainerRequest(storedContainer1); amClient.addContainerRequest(storedContainer2); @@@ -297,7 -302,7 +302,7 @@@ amClient.addContainerRequest(storedContainer11); amClient.addContainerRequest(storedContainer33); amClient.addContainerRequest(storedContainer43); -- ++ // test matching of containers List<? extends Collection<ContainerRequest>> matches; ContainerRequest storedRequest; @@@ -327,7 -332,7 +332,7 @@@ storedRequest = iter.next(); assertEquals(storedContainer33, storedRequest); amClient.removeContainerRequest(storedContainer33); -- ++ // exact matching with order maintained Resource testCapability2 = Resource.newInstance(2000, 1); matches = amClient.getMatchingRequests(priority, node, testCapability2); @@@ -342,12 -347,12 +347,12 @@@ } } amClient.removeContainerRequest(storedContainer6); -- ++ // matching with larger container. all requests returned Resource testCapability3 = Resource.newInstance(4000, 4); matches = amClient.getMatchingRequests(priority, node, testCapability3); assert(matches.size() == 4); -- ++ Resource testCapability4 = Resource.newInstance(1024, 2); matches = amClient.getMatchingRequests(priority, node, testCapability4); assert(matches.size() == 2); @@@ -357,14 -362,14 +362,14 @@@ ContainerRequest testRequest = testSet.iterator().next(); assertTrue(testRequest != storedContainer4); assertTrue(testRequest != storedContainer5); -- assert(testRequest == storedContainer2 || -- testRequest == storedContainer3); ++ assert(testRequest == storedContainer2 || ++ testRequest == storedContainer3); } -- ++ Resource testCapability5 = Resource.newInstance(512, 4); matches = amClient.getMatchingRequests(priority, node, testCapability5); assert(matches.size() == 0); -- ++ // verify requests without relaxed locality are only returned at specific // locations Resource testCapability7 = Resource.newInstance(2000, 1); @@@ -373,7 -378,7 +378,7 @@@ assert(matches.size() == 0); matches = amClient.getMatchingRequests(priority2, node, testCapability7); assert(matches.size() == 1); -- ++ amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null); @@@ -516,14 -521,14 +521,14 @@@ } } } -- ++ private void verifyMatches( -- List<? extends Collection<ContainerRequest>> matches, -- int matchSize) { ++ List<? extends Collection<ContainerRequest>> matches, ++ int matchSize) { assertEquals(1, matches.size()); assertEquals(matches.get(0).size(), matchSize); } -- ++ @Test (timeout=60000) public void testAMRMClientMatchingFitInferredRack() throws YarnException, IOException { AMRMClientImpl<ContainerRequest> amClient = null; @@@ -533,10 -538,10 +538,10 @@@ amClient.init(conf); amClient.start(); amClient.registerApplicationMaster("Host", 10000, ""); -- ++ Resource capability = Resource.newInstance(1024, 2); -- ContainerRequest storedContainer1 = ++ ContainerRequest storedContainer1 = new ContainerRequest(capability, nodes, null, priority); amClient.addContainerRequest(storedContainer1); @@@ -553,12 -558,12 +558,12 @@@ verifyMatches(matches, 1); storedRequest = matches.get(0).iterator().next(); assertEquals(storedContainer1, storedRequest); -- ++ // inferred rack match no longer valid after request is removed amClient.removeContainerRequest(storedContainer1); matches = amClient.getMatchingRequests(priority, rack, capability); assertTrue(matches.isEmpty()); -- ++ amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null); @@@ -576,24 -581,24 +581,24 @@@ // start am rm client amClient = (AMRMClientImpl<ContainerRequest>) AMRMClient -- .<ContainerRequest> createAMRMClient(); ++ .<ContainerRequest> createAMRMClient(); amClient.init(conf); amClient.start(); amClient.registerApplicationMaster("Host", 10000, ""); -- ++ Priority priority1 = Records.newRecord(Priority.class); priority1.setPriority(2); -- -- ContainerRequest storedContainer1 = ++ ++ ContainerRequest storedContainer1 = new ContainerRequest(capability, nodes, racks, priority); -- ContainerRequest storedContainer2 = ++ ContainerRequest storedContainer2 = new ContainerRequest(capability, nodes, racks, priority); -- ContainerRequest storedContainer3 = ++ ContainerRequest storedContainer3 = new ContainerRequest(capability, null, null, priority1); amClient.addContainerRequest(storedContainer1); amClient.addContainerRequest(storedContainer2); amClient.addContainerRequest(storedContainer3); -- ++ // test addition and storage RemoteRequestsTable<ContainerRequest> remoteRequestsTable = amClient.getTable(0); @@@ -604,21 -609,21 +609,21 @@@ containersRequestedAny = remoteRequestsTable.get(priority1, ResourceRequest.ANY, ExecutionType.GUARANTEED, capability) .remoteRequest.getNumContainers(); -- assertEquals(1, containersRequestedAny); -- List<? extends Collection<ContainerRequest>> matches = ++ assertEquals(1, containersRequestedAny); ++ List<? extends Collection<ContainerRequest>> matches = amClient.getMatchingRequests(priority, node, capability); verifyMatches(matches, 2); matches = amClient.getMatchingRequests(priority, rack, capability); verifyMatches(matches, 2); -- matches = ++ matches = amClient.getMatchingRequests(priority, ResourceRequest.ANY, capability); verifyMatches(matches, 2); matches = amClient.getMatchingRequests(priority1, rack, capability); assertTrue(matches.isEmpty()); -- matches = ++ matches = amClient.getMatchingRequests(priority1, ResourceRequest.ANY, capability); verifyMatches(matches, 1); -- ++ // test removal amClient.removeContainerRequest(storedContainer3); matches = amClient.getMatchingRequests(priority, node, capability); @@@ -628,20 -633,20 +633,20 @@@ verifyMatches(matches, 1); matches = amClient.getMatchingRequests(priority, rack, capability); verifyMatches(matches, 1); -- ++ // test matching of containers ContainerRequest storedRequest = matches.get(0).iterator().next(); assertEquals(storedContainer1, storedRequest); amClient.removeContainerRequest(storedContainer1); -- matches = ++ matches = amClient.getMatchingRequests(priority, ResourceRequest.ANY, capability); assertTrue(matches.isEmpty()); -- matches = ++ matches = amClient.getMatchingRequests(priority1, ResourceRequest.ANY, capability); assertTrue(matches.isEmpty()); // 0 requests left. everything got cleaned up assertTrue(amClient.getTable(0).isEmpty()); -- ++ // go through an exemplary allocation, matching and release cycle amClient.addContainerRequest(storedContainer1); amClient.addContainerRequest(storedContainer3); @@@ -655,16 -660,16 +660,16 @@@ AllocateResponse allocResponse = amClient.allocate(0.1f); assertEquals(0, amClient.ask.size()); assertEquals(0, amClient.release.size()); -- ++ assertEquals(nodeCount, amClient.getClusterNodeCount()); allocatedContainerCount += allocResponse.getAllocatedContainers().size(); for(Container container : allocResponse.getAllocatedContainers()) { -- ContainerRequest expectedRequest = ++ ContainerRequest expectedRequest = container.getPriority().equals(storedContainer1.getPriority()) ? storedContainer1 : storedContainer3; -- matches = amClient.getMatchingRequests(container.getPriority(), -- ResourceRequest.ANY, -- container.getResource()); ++ matches = amClient.getMatchingRequests(container.getPriority(), ++ ResourceRequest.ANY, ++ container.getResource()); // test correct matched container is returned verifyMatches(matches, 1); ContainerRequest matchedRequest = matches.get(0).iterator().next(); @@@ -678,7 -683,7 +683,7 @@@ triggerSchedulingWithNMHeartBeat(); } } -- ++ assertEquals(2, allocatedContainerCount); AllocateResponse allocResponse = amClient.allocate(0.1f); assertEquals(0, amClient.release.size()); @@@ -686,7 -691,7 +691,7 @@@ assertEquals(0, allocResponse.getAllocatedContainers().size()); // 0 requests left. everything got cleaned up assertTrue(remoteRequestsTable.isEmpty()); -- ++ amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null); @@@ -724,46 -729,46 +729,46 @@@ // start am rm client amClient = (AMRMClientImpl<ContainerRequest>) AMRMClient -- .<ContainerRequest> createAMRMClient(); ++ .<ContainerRequest> createAMRMClient(); amClient.init(conf); amClient.start(); amClient.registerApplicationMaster("Host", 10000, ""); -- ++ assertEquals(0, amClient.ask.size()); assertEquals(0, amClient.release.size()); -- -- ContainerRequest storedContainer1 = ++ ++ ContainerRequest storedContainer1 = new ContainerRequest(capability, nodes, racks, priority); amClient.addContainerRequest(storedContainer1); assertEquals(3, amClient.ask.size()); assertEquals(0, amClient.release.size()); -- ++ List<String> localNodeBlacklist = new ArrayList<String>(); localNodeBlacklist.add(node); -- ++ // put node in black list, so no container assignment amClient.updateBlacklist(localNodeBlacklist, null); int allocatedContainerCount = getAllocatedContainersNumber(amClient, -- DEFAULT_ITERATION); ++ DEFAULT_ITERATION); // the only node is in blacklist, so no allocation assertEquals(0, allocatedContainerCount); // Remove node from blacklist, so get assigned with 2 amClient.updateBlacklist(null, localNodeBlacklist); -- ContainerRequest storedContainer2 = -- new ContainerRequest(capability, nodes, racks, priority); ++ ContainerRequest storedContainer2 = ++ new ContainerRequest(capability, nodes, racks, priority); amClient.addContainerRequest(storedContainer2); allocatedContainerCount = getAllocatedContainersNumber(amClient, DEFAULT_ITERATION); assertEquals(2, allocatedContainerCount); -- ++ // Test in case exception in allocate(), blacklist is kept assertTrue(amClient.blacklistAdditions.isEmpty()); assertTrue(amClient.blacklistRemovals.isEmpty()); -- ++ // create a invalid ContainerRequest - memory value is minus -- ContainerRequest invalidContainerRequest = ++ ContainerRequest invalidContainerRequest = new ContainerRequest(Resource.newInstance(-1024, 1), nodes, racks, priority); amClient.addContainerRequest(invalidContainerRequest); @@@ -781,7 -786,7 +786,7 @@@ } } } -- ++ @Test (timeout=60000) public void testAMRMClientWithBlacklist() throws YarnException, IOException { AMRMClientImpl<ContainerRequest> amClient = null; @@@ -789,12 -794,12 +794,12 @@@ // start am rm client amClient = (AMRMClientImpl<ContainerRequest>) AMRMClient -- .<ContainerRequest> createAMRMClient(); ++ .<ContainerRequest> createAMRMClient(); amClient.init(conf); amClient.start(); amClient.registerApplicationMaster("Host", 10000, ""); String[] nodes = {"node1", "node2", "node3"}; -- ++ // Add nodes[0] and nodes[1] List<String> nodeList01 = new ArrayList<String>(); nodeList01.add(nodes[0]); @@@ -802,7 -807,7 +807,7 @@@ amClient.updateBlacklist(nodeList01, null); assertEquals(2, amClient.blacklistAdditions.size()); assertEquals(0, amClient.blacklistRemovals.size()); -- ++ // Add nodes[0] again, verify it is not added duplicated. List<String> nodeList02 = new ArrayList<String>(); nodeList02.add(nodes[0]); @@@ -810,7 -815,7 +815,7 @@@ amClient.updateBlacklist(nodeList02, null); assertEquals(3, amClient.blacklistAdditions.size()); assertEquals(0, amClient.blacklistRemovals.size()); -- ++ // Add nodes[1] and nodes[2] to removal list, // Verify addition list remove these two nodes. List<String> nodeList12 = new ArrayList<String>(); @@@ -819,7 -824,7 +824,7 @@@ amClient.updateBlacklist(null, nodeList12); assertEquals(1, amClient.blacklistAdditions.size()); assertEquals(2, amClient.blacklistRemovals.size()); -- ++ // Add nodes[1] again to addition list, // Verify removal list will remove this node. List<String> nodeList1 = new ArrayList<String>(); @@@ -844,10 -849,10 +849,10 @@@ AllocateResponse allocResponse = amClient.allocate(0.1f); assertEquals(0, amClient.ask.size()); assertEquals(0, amClient.release.size()); -- ++ assertEquals(nodeCount, amClient.getClusterNodeCount()); allocatedContainerCount += allocResponse.getAllocatedContainers().size(); -- ++ if(allocatedContainerCount == 0) { // let NM heartbeat to RM and trigger allocations triggerSchedulingWithNMHeartBeat(); @@@ -876,7 -892,7 +892,7 @@@ //setting an instance NMTokenCache amClient.setNMTokenCache(new NMTokenCache()); //asserting we are not using the singleton instance cache -- Assert.assertNotSame(NMTokenCache.getSingleton(), ++ Assert.assertNotSame(NMTokenCache.getSingleton(), amClient.getNMTokenCache()); amClient.init(conf); @@@ -899,7 -915,7 +915,7 @@@ } } } -- ++ @Test(timeout=30000) public void testAskWithNodeLabels() { AMRMClientImpl<ContainerRequest> client = @@@ -920,7 -936,7 +936,7 @@@ Assert.assertEquals(1, client.ask.size()); Assert.assertEquals("a", client.ask.iterator().next() .getNodeLabelExpression()); -- ++ // add exp=x to ANY, rack and node, only resource request has ANY resource // name will be assigned the label expression // add exp=x then add exp=a to ANY in same priority, only exp=a should kept @@@ -947,7 -963,7 +963,7 @@@ } } } -- ++ private void verifyAddRequestFailed(AMRMClient<ContainerRequest> client, ContainerRequest request) { try { @@@ -957,7 -973,7 +973,7 @@@ } Assert.fail(); } -- ++ @Test(timeout=30000) public void testAskWithInvalidNodeLabels() { AMRMClientImpl<ContainerRequest> client = @@@ -1130,10 -1146,10 +1146,10 @@@ private void testAllocation(final AMRMClientImpl<ContainerRequest> amClient) throws YarnException, IOException { // setup container request -- ++ assertEquals(0, amClient.ask.size()); assertEquals(0, amClient.release.size()); -- ++ amClient.addContainerRequest( new ContainerRequest(capability, nodes, racks, priority)); amClient.addContainerRequest( @@@ -1154,17 -1170,17 +1170,17 @@@ int allocatedContainerCount = 0; int iterationsLeft = 3; Set<ContainerId> releases = new TreeSet<ContainerId>(); -- ++ amClient.getNMTokenCache().clearCache(); Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache()); HashMap<String, Token> receivedNMTokens = new HashMap<String, Token>(); -- ++ while (allocatedContainerCount < containersRequestedAny && iterationsLeft-- > 0) { AllocateResponse allocResponse = amClient.allocate(0.1f); assertEquals(0, amClient.ask.size()); assertEquals(0, amClient.release.size()); -- ++ assertEquals(nodeCount, amClient.getClusterNodeCount()); allocatedContainerCount += allocResponse.getAllocatedContainers().size(); for(Container container : allocResponse.getAllocatedContainers()) { @@@ -1172,29 -1188,29 +1188,29 @@@ releases.add(rejectContainerId); amClient.releaseAssignedContainer(rejectContainerId); } -- ++ for (NMToken token : allocResponse.getNMTokens()) { String nodeID = token.getNodeId().toString(); if (receivedNMTokens.containsKey(nodeID)) { -- Assert.fail("Received token again for : " + nodeID); ++ Assert.fail("Received token again for : " + nodeID); } receivedNMTokens.put(nodeID, token.getToken()); } -- ++ if(allocatedContainerCount < containersRequestedAny) { // let NM heartbeat to RM and trigger allocations triggerSchedulingWithNMHeartBeat(); } } -- ++ // Should receive atleast 1 token Assert.assertTrue(receivedNMTokens.size() > 0 && receivedNMTokens.size() <= nodeCount); -- ++ assertEquals(allocatedContainerCount, containersRequestedAny); assertEquals(2, amClient.release.size()); assertEquals(0, amClient.ask.size()); -- ++ // need to tell the AMRMClient that we dont need these resources anymore amClient.removeContainerRequest( new ContainerRequest(capability, nodes, racks, priority)); @@@ -1204,7 -1220,7 +1220,7 @@@ // send 0 container count request for resources that are no longer needed ResourceRequest snoopRequest = amClient.ask.iterator().next(); assertEquals(0, snoopRequest.getNumContainers()); -- ++ // test RPC exception handling amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority)); @@@ -1212,7 -1228,7 +1228,7 @@@ racks, priority)); snoopRequest = amClient.ask.iterator().next(); assertEquals(2, snoopRequest.getNumContainers()); -- ++ ApplicationMasterProtocol realRM = amClient.rmClient; try { ApplicationMasterProtocol mockRM = mock(ApplicationMasterProtocol.class); @@@ -1221,8 -1237,8 +1237,8 @@@ public AllocateResponse answer(InvocationOnMock invocation) throws Exception { amClient.removeContainerRequest( -- new ContainerRequest(capability, nodes, -- racks, priority)); ++ new ContainerRequest(capability, nodes, ++ racks, priority)); amClient.removeContainerRequest( new ContainerRequest(capability, nodes, racks, priority)); throw new Exception(); @@@ -1398,7 -1414,7 +1414,7 @@@ } } } -- ++ private void sleep(int sleepTime) { try { Thread.sleep(sleepTime); @@@ -1414,7 -1430,7 +1430,7 @@@ try { AMRMTokenSecretManager amrmTokenSecretManager = yarnCluster.getResourceManager().getRMContext() -- .getAMRMTokenSecretManager(); ++ .getAMRMTokenSecretManager(); // start am rm client amClient = AMRMClient.<ContainerRequest> createAMRMClient(); @@@ -1429,7 -1445,7 +1445,7 @@@ getAMRMToken(); Assert.assertNotNull(amrmToken_1); Assert.assertEquals(amrmToken_1.decodeIdentifier().getKeyId(), -- amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId()); ++ amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId()); // Wait for enough time and make sure the roll_over happens // At mean time, the old AMRMToken should continue to work @@@ -1444,41 -1460,41 +1460,41 @@@ getAMRMToken(); Assert.assertNotNull(amrmToken_2); Assert.assertEquals(amrmToken_2.decodeIdentifier().getKeyId(), -- amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId()); ++ amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId()); Assert.assertNotEquals(amrmToken_1, amrmToken_2); // can do the allocate call with latest AMRMToken AllocateResponse response = amClient.allocate(0.1f); -- ++ // Verify latest AMRMToken can be used to send allocation request. UserGroupInformation testUser1 = UserGroupInformation.createRemoteUser("testUser1"); -- -- AMRMTokenIdentifierForTest newVersionTokenIdentifier = ++ ++ AMRMTokenIdentifierForTest newVersionTokenIdentifier = new AMRMTokenIdentifierForTest(amrmToken_2.decodeIdentifier(), "message"); -- ++ Assert.assertEquals("Message is changed after set to newVersionTokenIdentifier", "message", newVersionTokenIdentifier.getMessage()); -- org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newVersionToken = ++ org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newVersionToken = new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> ( -- newVersionTokenIdentifier.getBytes(), ++ newVersionTokenIdentifier.getBytes(), amrmTokenSecretManager.retrievePassword(newVersionTokenIdentifier), newVersionTokenIdentifier.getKind(), new Text()); -- ++ SecurityUtil.setTokenService(newVersionToken, yarnCluster -- .getResourceManager().getApplicationMasterService().getBindAddress()); ++ .getResourceManager().getApplicationMasterService().getBindAddress()); testUser1.addToken(newVersionToken); -- ++ AllocateRequest request = Records.newRecord(AllocateRequest.class); request.setResponseId(response.getResponseId()); testUser1.doAs(new PrivilegedAction<ApplicationMasterProtocol>() { @Override public ApplicationMasterProtocol run() { return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy( -- ApplicationMasterProtocol.class, -- yarnCluster.getResourceManager().getApplicationMasterService() -- .getBindAddress(), conf); ++ ApplicationMasterProtocol.class, ++ yarnCluster.getResourceManager().getApplicationMasterService() ++ .getBindAddress(), conf); } }).allocate(request); @@@ -1486,12 -1502,12 +1502,12 @@@ // and can not use this rolled-over token to make a allocate all. while (true) { if (amrmToken_2.decodeIdentifier().getKeyId() != amrmTokenSecretManager -- .getCurrnetMasterKeyData().getMasterKey().getKeyId()) { ++ .getCurrnetMasterKeyData().getMasterKey().getKeyId()) { if (amrmTokenSecretManager.getNextMasterKeyData() == null) { break; } else if (amrmToken_2.decodeIdentifier().getKeyId() != amrmTokenSecretManager.getNextMasterKeyData().getMasterKey() -- .getKeyId()) { ++ .getKeyId()) { break; } } @@@ -1503,27 -1519,27 +1519,27 @@@ UserGroupInformation testUser2 = UserGroupInformation.createRemoteUser("testUser2"); SecurityUtil.setTokenService(amrmToken_2, yarnCluster -- .getResourceManager().getApplicationMasterService().getBindAddress()); ++ .getResourceManager().getApplicationMasterService().getBindAddress()); testUser2.addToken(amrmToken_2); testUser2.doAs(new PrivilegedAction<ApplicationMasterProtocol>() { @Override public ApplicationMasterProtocol run() { return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy( -- ApplicationMasterProtocol.class, -- yarnCluster.getResourceManager().getApplicationMasterService() -- .getBindAddress(), conf); ++ ApplicationMasterProtocol.class, ++ yarnCluster.getResourceManager().getApplicationMasterService() ++ .getBindAddress(), conf); } }).allocate(Records.newRecord(AllocateRequest.class)); Assert.fail("The old Token should not work"); } catch (Exception ex) { Assert.assertTrue(ex instanceof InvalidToken); Assert.assertTrue(ex.getMessage().contains( -- "Invalid AMRMToken from " -- + amrmToken_2.decodeIdentifier().getApplicationAttemptId())); ++ "Invalid AMRMToken from " ++ + amrmToken_2.decodeIdentifier().getApplicationAttemptId())); } amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, -- null, null); ++ null, null); } finally { if (amClient != null && amClient.getServiceState() == STATE.STARTED) { @@@ -1534,7 -1550,7 +1550,7 @@@ @SuppressWarnings("unchecked") private org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> -- getAMRMToken() throws IOException { ++ getAMRMToken() throws IOException { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); Iterator<org.apache.hadoop.security.token.Token<?>> iter = http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc65104/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml ---------------------------------------------------------------------- diff --cc hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml index ae7ab0d,40eb54d..d42acaa --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml @@@ -16,7 -16,7 +16,7 @@@ --> <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; -- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> ++ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> <parent> <artifactId>hadoop-yarn</artifactId> <groupId>org.apache.hadoop</groupId> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
