Merge branch 'trunk' into HDFS-7240
Conflicts:
hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86414507
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86414507
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86414507
Branch: refs/heads/HDFS-7240
Commit: 86414507f4f584ddde97ff5f1d9e38204b36f77a
Parents: f713452 54fd0e4
Author: Anu Engineer <[email protected]>
Authored: Mon May 8 22:42:21 2017 -0700
Committer: Anu Engineer <[email protected]>
Committed: Mon May 8 22:42:21 2017 -0700
----------------------------------------------------------------------
dev-support/docker/Dockerfile | 2 +
hadoop-build-tools/pom.xml | 10 +
.../hadoop-client-minicluster/pom.xml | 4 -
hadoop-common-project/hadoop-common/pom.xml | 5 +
.../src/main/conf/log4j.properties | 14 +
.../org/apache/hadoop/conf/ConfigRedactor.java | 5 +-
.../hadoop/crypto/CryptoOutputStream.java | 4 +-
.../fs/CommonConfigurationKeysPublic.java | 17 +-
.../java/org/apache/hadoop/fs/CreateFlag.java | 8 +-
.../apache/hadoop/fs/FSDataOutputStream.java | 10 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 2 +-
.../java/org/apache/hadoop/fs/FileUtil.java | 5 +-
.../apache/hadoop/fs/RawLocalFileSystem.java | 2 +-
.../apache/hadoop/fs/StreamCapabilities.java | 67 ++
.../hadoop/fs/http/AbstractHttpFileSystem.java | 153 +++
.../apache/hadoop/fs/http/HttpFileSystem.java | 28 +
.../apache/hadoop/fs/http/HttpsFileSystem.java | 28 +
.../org/apache/hadoop/fs/http/package-info.java | 23 +
.../java/org/apache/hadoop/ha/NodeFencer.java | 3 +-
.../org/apache/hadoop/ha/PowerShellFencer.java | 154 +++
.../org/apache/hadoop/http/HttpServer2.java | 48 +-
.../hadoop/io/erasurecode/CodecRegistry.java | 176 +++
.../apache/hadoop/io/erasurecode/CodecUtil.java | 82 +-
.../apache/hadoop/io/erasurecode/ECSchema.java | 40 +-
.../io/erasurecode/ErasureCodeConstants.java | 1 +
.../rawcoder/DummyRawErasureCoderFactory.java | 12 +
.../NativeRSRawErasureCoderFactory.java | 13 +
.../NativeXORRawErasureCoderFactory.java | 13 +
.../rawcoder/RSLegacyRawDecoder.java | 270 +++++
.../rawcoder/RSLegacyRawEncoder.java | 129 +++
.../RSLegacyRawErasureCoderFactory.java | 51 +
.../rawcoder/RSRawDecoderLegacy.java | 270 -----
.../rawcoder/RSRawEncoderLegacy.java | 129 ---
.../rawcoder/RSRawErasureCoderFactory.java | 13 +
.../RSRawErasureCoderFactoryLegacy.java | 38 -
.../rawcoder/RawErasureCoderFactory.java | 12 +
.../rawcoder/XORRawErasureCoderFactory.java | 13 +
.../org/apache/hadoop/ipc/CallQueueManager.java | 1 +
.../apache/hadoop/ipc/DecayRpcScheduler.java | 31 +-
.../apache/hadoop/ipc/DefaultRpcScheduler.java | 4 +
.../org/apache/hadoop/ipc/RpcScheduler.java | 2 +
.../apache/hadoop/metrics2/AbstractMetric.java | 16 +-
.../org/apache/hadoop/metrics2/MetricsTag.java | 13 +-
.../metrics2/impl/AbstractMetricsRecord.java | 16 +-
.../org/apache/hadoop/metrics2/impl/MsInfo.java | 9 +-
.../hadoop/metrics2/lib/MetricsInfoImpl.java | 11 +-
.../hadoop/metrics2/lib/MetricsRegistry.java | 20 +-
.../apache/hadoop/metrics2/package-info.java | 2 +-
.../hadoop/metrics2/source/JvmMetricsInfo.java | 9 +-
.../hadoop/metrics2/util/MetricsCache.java | 18 +-
.../org/apache/hadoop/net/NetworkTopology.java | 12 +-
.../apache/hadoop/security/ssl/SSLFactory.java | 13 +-
.../hadoop/service/ServiceStateException.java | 66 +-
.../launcher/AbstractLaunchableService.java | 78 ++
.../HadoopUncaughtExceptionHandler.java | 129 +++
.../service/launcher/InterruptEscalator.java | 216 ++++
.../hadoop/service/launcher/IrqHandler.java | 178 +++
.../service/launcher/LaunchableService.java | 95 ++
.../service/launcher/LauncherArguments.java | 59 +
.../service/launcher/LauncherExitCodes.java | 183 +++
.../launcher/ServiceLaunchException.java | 81 ++
.../service/launcher/ServiceLauncher.java | 1044 ++++++++++++++++++
.../service/launcher/ServiceShutdownHook.java | 112 ++
.../hadoop/service/launcher/package-info.java | 462 ++++++++
.../apache/hadoop/util/ExitCodeProvider.java | 35 +
.../java/org/apache/hadoop/util/ExitUtil.java | 257 ++++-
.../hadoop/util/GenericOptionsParser.java | 125 ++-
.../org/apache/hadoop/util/StringUtils.java | 52 +-
.../org/apache/hadoop/util/SysInfoWindows.java | 2 +-
.../services/org.apache.hadoop.fs.FileSystem | 2 +
....erasurecode.rawcoder.RawErasureCoderFactory | 18 +
.../src/main/resources/core-default.xml | 35 +-
.../src/site/markdown/DeprecatedProperties.md | 1 +
.../src/site/markdown/FileSystemShell.md | 1 -
.../hadoop-common/src/site/markdown/Metrics.md | 4 +-
.../src/site/markdown/RackAwareness.md | 12 +-
.../apache/hadoop/conf/TestConfigRedactor.java | 22 +-
.../kms/TestLoadBalancingKMSClientProvider.java | 2 +-
.../apache/hadoop/fs/TestLocalFileSystem.java | 37 +-
.../apache/hadoop/fs/TestSymlinkLocalFS.java | 6 +-
.../contract/rawlocal/RawlocalFSContract.java | 2 +-
.../hadoop/fs/http/TestHttpFileSystem.java | 67 ++
.../fs/viewfs/ViewFileSystemBaseTest.java | 6 +-
.../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 2 +-
.../apache/hadoop/http/TestSSLHttpServer.java | 21 +-
.../erasurecode/TestCodecRawCoderMapping.java | 20 +-
.../io/erasurecode/TestCodecRegistry.java | 170 +++
.../hadoop/io/erasurecode/TestECSchema.java | 47 +-
.../coder/TestHHXORErasureCoder.java | 2 +-
.../erasurecode/coder/TestRSErasureCoder.java | 2 +-
.../rawcoder/RawErasureCoderBenchmark.java | 2 +-
.../erasurecode/rawcoder/TestDummyRawCoder.java | 4 +-
.../rawcoder/TestNativeRSRawCoder.java | 4 +-
.../rawcoder/TestNativeXORRawCoder.java | 4 +-
.../rawcoder/TestRSLegacyRawCoder.java | 33 +
.../io/erasurecode/rawcoder/TestRSRawCoder.java | 4 +-
.../rawcoder/TestRSRawCoderInteroperable1.java | 4 +-
.../rawcoder/TestRSRawCoderInteroperable2.java | 4 +-
.../rawcoder/TestRSRawCoderLegacy.java | 33 -
.../erasurecode/rawcoder/TestRawCoderBase.java | 16 +-
.../erasurecode/rawcoder/TestXORRawCoder.java | 4 +-
.../rawcoder/TestXORRawCoderInteroperable1.java | 4 +-
.../rawcoder/TestXORRawCoderInteroperable2.java | 4 +-
.../java/org/apache/hadoop/ipc/TestIPC.java | 2 +-
.../hadoop/security/ssl/TestSSLFactory.java | 10 +-
.../apache/hadoop/service/BreakableService.java | 23 +-
.../AbstractServiceLauncherTestBase.java | 317 ++++++
.../launcher/ExitTrackingServiceLauncher.java | 59 +
.../service/launcher/TestServiceConf.java | 146 +++
.../launcher/TestServiceInterruptHandling.java | 118 ++
.../service/launcher/TestServiceLauncher.java | 213 ++++
.../TestServiceLauncherCreationFailures.java | 83 ++
.../TestServiceLauncherInnerMethods.java | 95 ++
.../ExceptionInExecuteLaunchableService.java | 96 ++
.../testservices/FailInConstructorService.java | 33 +
.../testservices/FailInInitService.java | 38 +
.../testservices/FailInStartService.java | 37 +
.../testservices/FailingStopInStartService.java | 47 +
.../testservices/FailureTestService.java | 55 +
.../InitInConstructorLaunchableService.java | 63 ++
.../testservices/LaunchableRunningService.java | 111 ++
.../testservices/NoArgsAllowedService.java | 64 ++
.../testservices/NullBindLaunchableService.java | 46 +
.../launcher/testservices/RunningService.java | 84 ++
.../StoppingInStartLaunchableService.java | 49 +
.../StringConstructorOnlyService.java | 39 +
.../apache/hadoop/test/GenericTestUtils.java | 34 +
.../src/test/resources/contract/rawlocal.xml | 32 +-
hadoop-common-project/hadoop-kms/pom.xml | 4 -
.../crypto/key/kms/server/KMSWebServer.java | 7 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 23 +-
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 15 +-
.../hadoop/hdfs/DFSStripedOutputStream.java | 43 +-
.../org/apache/hadoop/hdfs/DataStreamer.java | 35 +-
.../hadoop/hdfs/DistributedFileSystem.java | 82 +-
.../apache/hadoop/hdfs/StripedDataStreamer.java | 10 +-
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 12 +
.../hdfs/client/HdfsClientConfigKeys.java | 4 +
.../hdfs/protocol/AddingECPolicyResponse.java | 66 ++
.../hdfs/protocol/ClientDatanodeProtocol.java | 5 +
.../hadoop/hdfs/protocol/ClientProtocol.java | 21 +-
.../hdfs/protocol/DatanodeVolumeInfo.java | 122 ++
.../hdfs/protocol/ErasureCodingPolicy.java | 53 +-
.../hadoop/hdfs/protocol/HdfsFileStatus.java | 22 +-
.../hdfs/protocol/IllegalECPolicyException.java | 34 +
.../datatransfer/DataTransferProtocol.java | 19 +-
.../hdfs/protocol/datatransfer/Sender.java | 29 +-
.../ClientDatanodeProtocolTranslatorPB.java | 29 +
.../ClientNamenodeProtocolTranslatorPB.java | 25 +
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 47 +
.../token/block/BlockTokenIdentifier.java | 66 +-
.../apache/hadoop/hdfs/util/ECPolicyLoader.java | 325 ++++++
.../src/main/proto/ClientDatanodeProtocol.proto | 9 +
.../src/main/proto/ClientNamenodeProtocol.proto | 3 +
.../src/main/proto/datatransfer.proto | 4 +
.../src/main/proto/erasurecoding.proto | 8 +
.../src/main/proto/hdfs.proto | 21 +
.../hdfs/protocol/TestErasureCodingPolicy.java | 86 ++
.../hadoop/hdfs/util/TestECPolicyLoader.java | 313 ++++++
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 8 -
.../fs/http/server/HttpFSServerWebServer.java | 7 +-
.../src/main/native/libhdfs-tests/expect.h | 18 +
.../libhdfs-tests/test_libhdfs_threaded.c | 16 +
.../src/main/native/libhdfs/exception.c | 75 +-
.../src/main/native/libhdfs/exception.h | 16 +-
.../src/main/native/libhdfs/hdfs.c | 90 ++
.../src/main/native/libhdfs/include/hdfs/hdfs.h | 61 +
.../src/main/native/libhdfs/jni_helper.c | 95 +-
.../src/main/native/libhdfs/jni_helper.h | 35 +
.../libhdfs/os/posix/thread_local_storage.c | 54 +-
.../native/libhdfs/os/thread_local_storage.h | 61 +-
.../libhdfs/os/windows/thread_local_storage.c | 50 +-
.../src/main/conf/user_ec_policies.xml.template | 71 ++
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 17 +-
.../hadoop/hdfs/net/DFSNetworkTopology.java | 24 +-
.../hadoop/hdfs/net/DFSTopologyNodeImpl.java | 137 +++
.../hdfs/protocol/datatransfer/Receiver.java | 20 +-
...tDatanodeProtocolServerSideTranslatorPB.java | 28 +
...tNamenodeProtocolServerSideTranslatorPB.java | 31 +-
.../hadoop/hdfs/qjournal/server/JNStorage.java | 25 +-
.../hadoop/hdfs/qjournal/server/Journal.java | 21 +-
.../hdfs/qjournal/server/JournalNodeSyncer.java | 104 +-
.../block/BlockPoolTokenSecretManager.java | 34 +-
.../token/block/BlockTokenSecretManager.java | 70 +-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 47 +-
.../hadoop/hdfs/server/balancer/KeyManager.java | 8 +-
.../hdfs/server/balancer/NameNodeConnector.java | 13 +-
.../server/blockmanagement/BlockManager.java | 44 +-
.../BlockPlacementPolicyDefault.java | 36 +-
.../blockmanagement/DatanodeDescriptor.java | 68 +-
.../server/blockmanagement/DatanodeManager.java | 26 +-
.../blockmanagement/LowRedundancyBlocks.java | 6 +-
.../apache/hadoop/hdfs/server/common/Util.java | 12 +-
.../hdfs/server/datanode/BPOfferService.java | 3 +-
.../hdfs/server/datanode/BPServiceActor.java | 3 +
.../hdfs/server/datanode/BlockReceiver.java | 12 +-
.../hadoop/hdfs/server/datanode/DNConf.java | 7 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 80 +-
.../server/datanode/DataNodeFaultInjector.java | 6 +-
.../hdfs/server/datanode/DataXceiver.java | 113 +-
.../hdfs/server/datanode/FileIoProvider.java | 4 +-
.../server/datanode/ProfilingFileIoEvents.java | 19 +-
.../erasurecode/ErasureCodingWorker.java | 3 +-
.../erasurecode/StripedBlockReader.java | 4 +-
.../erasurecode/StripedBlockWriter.java | 11 +-
.../erasurecode/StripedReconstructionInfo.java | 16 +-
.../datanode/erasurecode/StripedWriter.java | 5 +-
.../AvailableSpaceVolumeChoosingPolicy.java | 20 +-
.../server/datanode/fsdataset/FsDatasetSpi.java | 6 +-
.../RoundRobinVolumeChoosingPolicy.java | 2 +-
.../fsdataset/VolumeChoosingPolicy.java | 5 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 21 +-
.../datanode/fsdataset/impl/FsVolumeList.java | 19 +-
.../datanode/web/SimpleHttpProxyHandler.java | 4 +-
.../namenode/ErasureCodingPolicyManager.java | 95 +-
.../server/namenode/FSDirErasureCodingOp.java | 13 +-
.../hdfs/server/namenode/FSDirSnapshotOp.java | 7 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 31 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 5 +-
.../hadoop/hdfs/server/namenode/FSImage.java | 52 +-
.../server/namenode/FSImageFormatPBINode.java | 3 +-
.../hdfs/server/namenode/FSNamesystem.java | 51 +-
.../hdfs/server/namenode/FSNamesystemLock.java | 121 +-
.../hdfs/server/namenode/INodeDirectory.java | 8 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 7 +-
.../hdfs/server/namenode/INodesInPath.java | 78 +-
.../hdfs/server/namenode/LeaseManager.java | 103 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 10 +-
.../server/namenode/ha/StandbyCheckpointer.java | 7 +-
.../snapshot/DirectorySnapshottableFeature.java | 15 +-
.../namenode/snapshot/SnapshotManager.java | 24 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 30 +-
.../org/apache/hadoop/hdfs/tools/ECAdmin.java | 63 ++
.../src/main/resources/hdfs-default.xml | 42 +-
.../src/main/webapps/hdfs/dfshealth.html | 2 +-
.../src/main/webapps/hdfs/dfshealth.js | 4 +-
.../src/site/markdown/HDFSCommands.md | 2 +
.../src/site/markdown/HDFSErasureCoding.md | 28 +-
.../org/apache/hadoop/TestRefreshCallQueue.java | 88 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 9 +-
.../hadoop/hdfs/TestBlockStoragePolicy.java | 143 +++
.../apache/hadoop/hdfs/TestDFSOutputStream.java | 25 +
.../hadoop/hdfs/TestDFSStripedInputStream.java | 2 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 37 +-
.../TestDFSStripedOutputStreamWithFailure.java | 2 +-
.../hadoop/hdfs/TestDataTransferProtocol.java | 3 +-
.../hadoop/hdfs/TestDistributedFileSystem.java | 39 +
.../apache/hadoop/hdfs/TestEncryptionZones.java | 10 +-
.../hadoop/hdfs/TestErasureCodingPolicies.java | 58 +
.../org/apache/hadoop/hdfs/TestGetBlocks.java | 77 +-
.../org/apache/hadoop/hdfs/TestHdfsAdmin.java | 33 +
.../hadoop/hdfs/TestReconstructStripedFile.java | 2 +-
.../TestUnsetAndChangeDirectoryEcPolicy.java | 2 +-
.../hdfs/TestWriteBlockGetsBlockLengthHint.java | 6 +-
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 35 +
.../hdfs/qjournal/TestJournalNodeSync.java | 1 +
.../security/token/block/TestBlockToken.java | 186 +++-
.../hdfs/server/balancer/TestBalancer.java | 100 +-
.../server/balancer/TestBalancerRPCDelay.java | 32 +
.../blockmanagement/BlockManagerTestUtil.java | 21 +
.../blockmanagement/TestSlowDiskTracker.java | 4 +-
.../server/datanode/BlockReportTestBase.java | 2 +-
.../server/datanode/SimulatedFSDataset.java | 19 +-
.../hdfs/server/datanode/TestBlockRecovery.java | 6 +-
.../server/datanode/TestBlockReplacement.java | 2 +-
.../server/datanode/TestDataNodeMXBean.java | 4 +-
.../server/datanode/TestDataNodeMetrics.java | 7 +-
.../datanode/TestDataNodeVolumeMetrics.java | 4 +-
.../TestDataXceiverLazyPersistHint.java | 4 +-
.../hdfs/server/datanode/TestDiskError.java | 5 +-
.../server/datanode/TestFsDatasetCache.java | 62 +-
.../server/datanode/TestSimulatedFSDataset.java | 4 +-
.../extdataset/ExternalDatasetImpl.java | 10 +-
.../TestAvailableSpaceVolumeChoosingPolicy.java | 76 +-
.../TestRoundRobinVolumeChoosingPolicy.java | 29 +-
.../fsdataset/impl/TestFsDatasetImpl.java | 4 +-
.../fsdataset/impl/TestFsVolumeList.java | 2 +-
.../fsdataset/impl/TestWriteToReplica.java | 29 +-
.../hdfs/server/namenode/NameNodeAdapter.java | 31 +-
.../hdfs/server/namenode/TestCheckpoint.java | 4 +-
.../namenode/TestDecommissioningStatus.java | 113 +-
.../TestDefaultBlockPlacementPolicy.java | 46 +
.../server/namenode/TestEnabledECPolicies.java | 11 +-
.../server/namenode/TestFSNamesystemLock.java | 16 +-
.../hdfs/server/namenode/TestLeaseManager.java | 286 ++++-
.../namenode/TestMetadataVersionOutput.java | 4 +-
.../namenode/TestNameNodeStatusMXBean.java | 5 +-
.../namenode/TestNamenodeStorageDirectives.java | 330 ++++++
.../namenode/TestReconstructStripedBlocks.java | 2 +
.../hdfs/server/namenode/TestStartup.java | 5 +-
.../namenode/ha/TestStandbyCheckpoints.java | 60 +
.../snapshot/TestOpenFilesWithSnapshot.java | 298 +++++
.../snapshot/TestSnapshotDiffReport.java | 188 +++-
.../namenode/snapshot/TestSnapshotManager.java | 11 +-
.../shortcircuit/TestShortCircuitCache.java | 8 +-
.../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 20 +
.../hadoop/tools/TestHdfsConfigFields.java | 2 -
.../test/resources/testErasureCodingConf.xml | 68 ++
.../src/test/resources/test_ec_policies.xml | 65 ++
hadoop-mapreduce-project/.gitignore | 1 -
hadoop-mapreduce-project/conf/mapred-site.xml | 21 +
.../conf/mapred-site.xml.template | 21 -
.../hadoop/mapred/LocalContainerLauncher.java | 46 +-
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 16 +-
.../java/org/apache/hadoop/mapred/JVMId.java | 2 +-
.../org/apache/hadoop/mapred/Operation.java | 14 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 6 +
.../mapreduce/v2/hs/HistoryFileManager.java | 5 -
.../mapreduce/v2/hs/JobHistoryServer.java | 3 -
.../org/apache/hadoop/mapred/YARNRunner.java | 132 ++-
.../apache/hadoop/hdfs/NNBenchWithoutMR.java | 6 +-
.../mapred/TestMRTimelineEventHandling.java | 2 +-
.../apache/hadoop/mapred/TestYARNRunner.java | 167 +++
.../hadoop/mapreduce/GrowingSleepJob.java | 68 ++
.../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 6 +-
.../apache/hadoop/test/MapredTestDriver.java | 3 +
.../org/apache/hadoop/examples/pi/Parser.java | 8 +-
hadoop-project/pom.xml | 24 +-
hadoop-tools/hadoop-archive-logs/pom.xml | 5 +
hadoop-tools/hadoop-aws/pom.xml | 10 +-
.../org/apache/hadoop/fs/s3a/Constants.java | 28 +-
.../hadoop/fs/s3a/S3AEncryptionMethods.java | 13 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 30 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 120 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 83 +-
.../site/markdown/tools/hadoop-aws/testing.md | 4 +-
.../fs/s3a/AbstractTestS3AEncryption.java | 4 +
.../ITestS3AEncryptionAlgorithmValidation.java | 4 +-
.../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 308 +++++-
.../hadoop/fs/s3a/TestSSEConfiguration.java | 223 ++++
.../src/site/markdown/index.md | 4 +-
.../fs/adl/live/AdlStorageConfiguration.java | 8 +-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 3 -
.../fs/azure/RemoteSASKeyGeneratorImpl.java | 26 +-
.../fs/azure/RemoteWasbAuthorizerImpl.java | 22 +-
.../fs/azure/SyncableDataOutputStream.java | 12 +-
.../fs/azure/security/WasbTokenRenewer.java | 6 -
.../mapred/gridmix/TestGridmixSubmission.java | 3 +-
.../hadoop/metrics2/impl/TestKafkaMetrics.java | 10 +-
hadoop-tools/hadoop-openstack/pom.xml | 6 +-
.../SwiftAuthenticationFailedException.java | 6 +-
.../exceptions/SwiftBadRequestException.java | 6 +-
.../SwiftInvalidResponseException.java | 9 +-
.../SwiftThrottledRequestException.java | 6 +-
.../apache/hadoop/fs/swift/http/CopyMethod.java | 41 -
.../hadoop/fs/swift/http/CopyRequest.java | 41 +
.../swift/http/HttpInputStreamWithRelease.java | 25 +-
.../hadoop/fs/swift/http/SwiftRestClient.java | 805 +++++++-------
.../snative/SwiftNativeFileSystemStore.java | 11 +-
.../hadoop/fs/swift/util/HttpResponseUtils.java | 121 ++
.../TestSwiftFileSystemPartitionedUploads.java | 2 +-
.../fs/swift/http/TestSwiftRestClient.java | 2 +-
.../src/test/resources/log4j.properties | 3 -
.../org/apache/hadoop/tools/rumen/TaskInfo.java | 29 +-
.../apache/hadoop/tools/rumen/ZombieJob.java | 9 +-
hadoop-tools/hadoop-sls/pom.xml | 19 +
hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 46 +-
.../hadoop/yarn/sls/ReservationClientUtil.java | 78 ++
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 859 +++++++++-----
.../hadoop/yarn/sls/appmaster/AMSimulator.java | 131 ++-
.../yarn/sls/appmaster/MRAMSimulator.java | 9 +-
.../sls/resourcemanager/MockAMLauncher.java | 5 +
.../sls/scheduler/SLSCapacityScheduler.java | 33 +-
.../yarn/sls/scheduler/SLSFairScheduler.java | 31 +-
.../yarn/sls/scheduler/SchedulerWrapper.java | 4 +
.../hadoop/yarn/sls/scheduler/TaskRunner.java | 9 +-
.../hadoop/yarn/sls/synthetic/SynthJob.java | 306 +++++
.../yarn/sls/synthetic/SynthJobClass.java | 180 +++
.../sls/synthetic/SynthTraceJobProducer.java | 316 ++++++
.../hadoop/yarn/sls/synthetic/SynthUtils.java | 101 ++
.../yarn/sls/synthetic/SynthWorkload.java | 121 ++
.../hadoop/yarn/sls/synthetic/package-info.java | 22 +
.../apache/hadoop/yarn/sls/utils/SLSUtils.java | 52 +-
.../apache/hadoop/yarn/sls/web/SLSWebApp.java | 33 +-
.../src/site/markdown/SchedulerLoadSimulator.md | 150 ++-
.../hadoop/yarn/sls/BaseSLSRunnerTest.java | 120 ++
.../apache/hadoop/yarn/sls/TestSLSRunner.java | 90 +-
.../hadoop/yarn/sls/TestSynthJobGeneration.java | 96 ++
.../yarn/sls/appmaster/TestAMSimulator.java | 2 +-
.../yarn/sls/scheduler/TestTaskRunner.java | 2 +-
.../hadoop/yarn/sls/utils/TestSLSUtils.java | 30 +
.../src/test/resources/capacity-scheduler.xml | 10 +
.../src/test/resources/fair-scheduler.xml | 8 +-
.../hadoop-sls/src/test/resources/inputsls.json | 55 +
.../hadoop-sls/src/test/resources/nodes.json | 84 ++
.../src/test/resources/sls-runner.xml | 6 +-
.../hadoop-sls/src/test/resources/syn.json | 53 +
.../hadoop-sls/src/test/resources/yarn-site.xml | 10 +-
.../yarn/api/ContainerManagementProtocol.java | 8 +-
.../yarn/api/records/QueueConfigurations.java | 150 +++
.../hadoop/yarn/api/records/QueueInfo.java | 42 +
.../hadoop/yarn/conf/YarnConfiguration.java | 4 +
.../yarn/exceptions/ConfigurationException.java | 45 +
.../src/main/proto/yarn_protos.proto | 14 +
.../yarn/client/api/impl/YarnClientImpl.java | 10 +-
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 58 +-
.../hadoop/yarn/client/ProtocolHATestBase.java | 6 +-
.../yarn/client/api/impl/TestAMRMClient.java | 2 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 4 +-
.../impl/pb/QueueConfigurationsPBImpl.java | 137 +++
.../api/records/impl/pb/QueueInfoPBImpl.java | 98 +-
.../apache/hadoop/yarn/client/ServerProxy.java | 5 +
.../yarn/client/api/impl/TimelineConnector.java | 9 +-
.../hadoop/yarn/event/AsyncDispatcher.java | 12 +-
.../apache/hadoop/yarn/event/Dispatcher.java | 9 -
.../hadoop/yarn/event/EventDispatcher.java | 17 +-
.../logaggregation/AggregatedLogFormat.java | 4 +
.../yarn/util/ProcfsBasedProcessTree.java | 37 +-
.../util/ResourceCalculatorProcessTree.java | 54 -
.../yarn/util/WindowsBasedProcessTree.java | 12 -
.../hadoop/yarn/webapp/util/WebAppUtils.java | 17 +-
.../src/main/resources/yarn-default.xml | 22 +-
.../hadoop/yarn/api/TestPBImplRecords.java | 21 +-
.../client/api/impl/TestTimelineClient.java | 2 +
.../hadoop/yarn/event/DrainDispatcher.java | 11 +-
.../yarn/util/TestProcfsBasedProcessTree.java | 72 +-
.../yarn/util/TestWindowsBasedProcessTree.java | 12 -
.../pom.xml | 12 +-
.../server/nodemanager/ContainerExecutor.java | 4 +-
.../nodemanager/DefaultContainerExecutor.java | 7 +-
.../nodemanager/LinuxContainerExecutor.java | 79 +-
.../nodemanager/NodeHealthCheckerService.java | 54 +-
.../yarn/server/nodemanager/NodeManager.java | 5 -
.../server/nodemanager/NodeStatusUpdater.java | 6 +
.../nodemanager/NodeStatusUpdaterImpl.java | 8 +-
.../nodemanager/amrmproxy/AMRMProxyService.java | 63 +-
.../containermanager/ContainerManager.java | 2 -
.../containermanager/ContainerManagerImpl.java | 28 -
.../launcher/ContainerLaunch.java | 38 +-
.../launcher/ContainerRelaunch.java | 9 +
.../linux/resources/CGroupsHandlerImpl.java | 10 +-
.../DelegatingLinuxContainerRuntime.java | 9 +-
.../JavaSandboxLinuxContainerRuntime.java | 13 +-
.../runtime/docker/DockerCommandExecutor.java | 191 ++++
.../linux/runtime/docker/package-info.java | 26 +
.../monitor/ContainersMonitorImpl.java | 49 +-
.../util/CgroupsLCEResourcesHandler.java | 13 +-
.../impl/container-executor.c | 40 +-
.../impl/container-executor.h | 22 +-
.../nodemanager/DummyContainerManager.java | 6 +-
.../TestDefaultContainerExecutor.java | 3 +-
.../nodemanager/TestLinuxContainerExecutor.java | 9 +-
.../TestLinuxContainerExecutorWithMocks.java | 217 ++--
.../nodemanager/TestNodeHealthService.java | 11 +-
.../nodemanager/TestNodeManagerResync.java | 164 ---
.../amrmproxy/BaseAMRMProxyTest.java | 21 +-
.../amrmproxy/TestAMRMProxyService.java | 65 ++
.../TestAMRMProxyTokenSecretManager.java | 81 ++
.../BaseContainerManagerTest.java | 13 +-
.../containermanager/TestContainerManager.java | 5 -
.../TestContainerManagerRecovery.java | 16 +-
.../containermanager/TestNMProxy.java | 30 +-
.../launcher/TestContainerLaunch.java | 65 +-
.../MockPrivilegedOperationCaptor.java | 68 ++
.../linux/resources/TestCGroupsHandlerImpl.java | 37 +
.../runtime/TestDockerContainerRuntime.java | 13 +-
.../TestJavaSandboxLinuxContainerRuntime.java | 20 +-
.../docker/TestDockerCommandExecutor.java | 218 ++++
.../runtime/docker/TestDockerLoadCommand.java | 48 +
.../runtime/docker/TestDockerRunCommand.java | 63 ++
.../TestResourceLocalizationService.java | 2 -
.../TestContainersMonitorResourceChange.java | 3 +-
.../TestContainerSchedulerQueuing.java | 9 +-
.../util/TestCgroupsLCEResourcesHandler.java | 29 +
...-container-executer-with-configuration-error | 20 +
.../hadoop-yarn-server-resourcemanager/pom.xml | 1 +
.../server/resourcemanager/AdminService.java | 3 +
.../ApplicationMasterService.java | 2 +-
.../server/resourcemanager/RMAppManager.java | 1 +
.../server/resourcemanager/ResourceManager.java | 1 -
.../ProportionalCapacityPreemptionPolicy.java | 17 +-
.../invariants/InvariantViolationException.java | 35 +
.../monitor/invariants/InvariantsChecker.java | 96 ++
.../invariants/MetricsInvariantChecker.java | 195 ++++
.../monitor/invariants/package-info.java | 22 +
.../recovery/ZKRMStateStore.java | 506 +++++++--
.../reservation/InMemoryPlan.java | 11 +
.../PeriodicRLESparseResourceAllocation.java | 167 +++
.../resourcemanager/reservation/PlanView.java | 9 +
.../RLESparseResourceAllocation.java | 51 +
.../planning/AlignedPlannerWithGreedy.java | 15 +-
.../planning/GreedyReservationAgent.java | 13 +-
.../reservation/planning/IterativePlanner.java | 196 ++--
.../reservation/planning/ReservationAgent.java | 23 +-
.../planning/SimpleCapacityReplanner.java | 8 +-
.../reservation/planning/StageAllocator.java | 10 +-
.../planning/StageAllocatorGreedy.java | 4 +-
.../planning/StageAllocatorGreedyRLE.java | 4 +-
.../planning/StageAllocatorLowCostAligned.java | 279 +++--
.../planning/StageEarliestStart.java | 46 -
.../planning/StageEarliestStartByDemand.java | 106 --
.../StageEarliestStartByJobArrival.java | 39 -
.../planning/StageExecutionInterval.java | 47 +
.../StageExecutionIntervalByDemand.java | 144 +++
.../StageExecutionIntervalUnconstrained.java | 73 ++
.../server/resourcemanager/rmapp/RMAppImpl.java | 24 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 6 +-
.../rmapp/attempt/RMAppAttemptMetrics.java | 2 +-
.../scheduler/AbstractYarnScheduler.java | 2 +-
.../resourcemanager/scheduler/NodeType.java | 12 +-
.../resourcemanager/scheduler/QueueMetrics.java | 21 +-
.../scheduler/capacity/AbstractCSQueue.java | 25 +
.../scheduler/capacity/CSQueueMetrics.java | 4 +-
.../capacity/CapacitySchedulerQueueManager.java | 6 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 3 +
.../scheduler/fair/FSQueueMetrics.java | 4 +-
.../scheduler/fair/FSSchedulerNode.java | 8 +-
.../yarn/server/resourcemanager/MockRM.java | 10 +-
.../server/resourcemanager/TestAppManager.java | 2 +
.../resourcemanager/TestClientRMService.java | 25 +
.../resourcemanager/TestRMDispatcher.java | 1 +
.../resourcemanager/TestRMStoreCommands.java | 15 +-
.../applicationsmanager/TestAMRestart.java | 140 ++-
.../invariants/TestMetricsInvariantChecker.java | 99 ++
.../recovery/RMStateStoreTestBase.java | 10 +-
.../recovery/TestFSRMStateStore.java | 1 +
.../recovery/TestLeveldbRMStateStore.java | 1 +
.../recovery/TestZKRMStateStore.java | 573 +++++++++-
.../reservation/ReservationSystemTestUtil.java | 14 +
...TestPeriodicRLESparseResourceAllocation.java | 142 +++
.../TestRLESparseResourceAllocation.java | 56 +-
.../planning/TestAlignedPlanner.java | 417 ++++++-
.../planning/TestGreedyReservationAgent.java | 14 +-
.../planning/TestSimpleCapacityReplanner.java | 4 +-
.../scheduler/TestAbstractYarnScheduler.java | 2 +-
.../capacity/TestCapacityScheduler.java | 15 +-
.../scheduler/fair/TestFairScheduler.java | 15 +
.../src/test/resources/invariants.txt | 54 +
.../hadoop/yarn/server/MiniYARNCluster.java | 22 +-
.../server/TestContainerManagerSecurity.java | 14 +-
.../hadoop/yarn/server/TestMiniYarnCluster.java | 34 +-
.../collector/AppLevelTimelineCollector.java | 8 +-
.../collector/TimelineCollector.java | 27 +-
.../storage/FileSystemTimelineReaderImpl.java | 129 +--
.../collector/TestTimelineCollector.java | 95 +-
.../src/site/markdown/FairScheduler.md | 6 +
.../src/main/webapp/app/adapters/yarn-app.js | 2 +-
.../webapp/app/components/em-table-html-cell.js | 23 +
.../main/webapp/app/components/timeline-view.js | 13 +-
.../webapp/app/controllers/app-table-columns.js | 213 ++--
.../webapp/app/controllers/yarn-app-attempt.js | 32 +-
.../webapp/app/controllers/yarn-app-attempts.js | 23 +-
.../src/main/webapp/app/controllers/yarn-app.js | 21 +-
.../webapp/app/controllers/yarn-apps/apps.js | 9 +-
.../app/controllers/yarn-apps/services.js | 9 +-
.../webapp/app/controllers/yarn-nodes/table.js | 8 +
.../webapp/app/controllers/yarn-services.js | 79 +-
.../src/main/webapp/app/models/yarn-app.js | 6 +-
.../src/main/webapp/app/router.js | 1 +
.../src/main/webapp/app/routes/yarn-services.js | 34 +
.../src/main/webapp/app/styles/app.css | 9 +
.../main/webapp/app/templates/application.hbs | 13 +-
.../templates/components/app-attempt-table.hbs | 8 +-
.../templates/components/container-table.hbs | 2 +-
.../templates/components/em-table-html-cell.hbs | 23 +
.../app/templates/components/timeline-view.hbs | 8 +-
.../webapp/app/templates/yarn-app-attempts.hbs | 8 +-
.../src/main/webapp/app/templates/yarn-app.hbs | 12 +-
.../src/main/webapp/app/templates/yarn-apps.hbs | 4 -
.../webapp/app/templates/yarn-apps/apps.hbs | 2 +-
.../webapp/app/templates/yarn-apps/services.hbs | 2 +-
.../webapp/app/templates/yarn-nodes/table.hbs | 2 +-
.../main/webapp/app/templates/yarn-services.hbs | 86 ++
.../src/main/webapp/app/utils/converter.js | 24 +
.../components/em-table-html-cell-test.js | 43 +
.../tests/unit/routes/yarn-services-test.js | 29 +
566 files changed, 23470 insertions(+), 4541 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86414507/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
----------------------------------------------------------------------
diff --cc hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index 1289115,22a2117..fb7eadf
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@@ -322,36 -322,16 +322,50 @@@ log4j.appender.EWMA.cleanupInterval=${y
log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds}
log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages}
++<<<<<<< HEAD
+# Fair scheduler requests log on state dump
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=DEBUG,FSLOGGER
+log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=false
+log4j.appender.FSLOGGER=org.apache.log4j.RollingFileAppender
+log4j.appender.FSLOGGER.File=${hadoop.log.dir}/fairscheduler-statedump.log
+log4j.appender.FSLOGGER.layout=org.apache.log4j.PatternLayout
+log4j.appender.FSLOGGER.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.FSLOGGER.MaxFileSize=${hadoop.log.maxfilesize}
+log4j.appender.FSLOGGER.MaxBackupIndex=${hadoop.log.maxbackupindex}
+
+#
+# Add a logger for ozone that is separate from the Datanode.
+#
+log4j.logger.org.apache.hadoop.ozone=DEBUG,OZONE,FILE
+
+# Do not log into datanode logs. Remove this line to have single log.
+log4j.additivity.org.apache.hadoop.ozone=false
+
+# For development purposes, log both to console and log file.
+log4j.appender.OZONE=org.apache.log4j.ConsoleAppender
+log4j.appender.OZONE.Threshold=info
+log4j.appender.OZONE.layout=org.apache.log4j.PatternLayout
+log4j.appender.OZONE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p \
+ %X{component} %X{function} %X{resource} %X{user} %X{request} - %m%n
+
+# Real ozone logger that writes to ozone.log
+log4j.appender.FILE=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.FILE.File=${hadoop.log.dir}/ozone.log
+log4j.appender.FILE.Threshold=debug
+log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
+log4j.appender.FILE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p \
+(%F:%L) %X{function} %X{resource} %X{user} %X{request} - \
+%m%n
+ #
+ # Fair scheduler state dump
+ #
+ # Use following logger to dump the state to a separate file
+
+
#log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=DEBUG,FSSTATEDUMP
+
#log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=false
+ #log4j.appender.FSSTATEDUMP=org.apache.log4j.RollingFileAppender
+ #log4j.appender.FSSTATEDUMP.File=${hadoop.log.dir}/fairscheduler-statedump.log
+ #log4j.appender.FSSTATEDUMP.layout=org.apache.log4j.PatternLayout
+ #log4j.appender.FSSTATEDUMP.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+ #log4j.appender.FSSTATEDUMP.MaxFileSize=${hadoop.log.maxfilesize}
-#log4j.appender.FSSTATEDUMP.MaxBackupIndex=${hadoop.log.maxbackupindex}
++#log4j.appender.FSSTATEDUMP.MaxBackupIndex=${hadoop.log.maxbackupindex}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86414507/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86414507/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86414507/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86414507/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86414507/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86414507/hadoop-project/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86414507/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
----------------------------------------------------------------------
diff --cc
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
index 4effb0d,ea08b2b..eb16fb5
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
@@@ -133,91 -122,92 +122,92 @@@ public class RemoteWasbAuthorizerImpl i
public boolean authorize(String wasbAbsolutePath, String accessType)
throws WasbAuthorizationException, IOException {
- try {
+ try {
/* Make an exception for the internal -RenamePending files */
- if
(wasbAbsolutePath.endsWith(NativeAzureFileSystem.FolderRenamePending.SUFFIX)) {
- return true;
- }
-
- setDelegationToken();
- URIBuilder uriBuilder = new URIBuilder(remoteAuthorizerServiceUrl);
- uriBuilder.setPath("/" + CHECK_AUTHORIZATION_OP);
- uriBuilder.addParameter(WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME,
- wasbAbsolutePath);
- uriBuilder.addParameter(ACCESS_OPERATION_QUERY_PARAM_NAME,
- accessType);
- if (isSecurityEnabled && StringUtils.isNotEmpty(delegationToken)) {
- uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME,
- delegationToken);
- }
-
- String responseBody = null;
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- UserGroupInformation connectUgi = ugi.getRealUser();
- if (connectUgi == null) {
- connectUgi = ugi;
- } else {
- uriBuilder.addParameter(Constants.DOAS_PARAM,
ugi.getShortUserName());
- }
-
- try {
- responseBody = connectUgi
- .doAs(new PrivilegedExceptionAction<String>() {
- @Override
- public String run() throws Exception {
- AuthenticatedURL.Token token = null;
- HttpGet httpGet = new HttpGet(uriBuilder.build());
- if (isKerberosSupportEnabled && UserGroupInformation
- .isSecurityEnabled() && (delegationToken == null
- || delegationToken.isEmpty())) {
- token = new AuthenticatedURL.Token();
- final Authenticator kerberosAuthenticator = new
KerberosDelegationTokenAuthenticator();
- try {
- kerberosAuthenticator
- .authenticate(uriBuilder.build().toURL(), token);
- Validate.isTrue(token.isSet(),
- "Authenticated Token is NOT present. The request
cannot proceed.");
- } catch (AuthenticationException e){
- throw new IOException("Authentication failed in check
authorization", e);
- }
- if (token != null) {
- httpGet.setHeader("Cookie",
- AuthenticatedURL.AUTH_COOKIE + "=" + token);
- }
+ if
(wasbAbsolutePath.endsWith(NativeAzureFileSystem.FolderRenamePending.SUFFIX)) {
+ return true;
+ }
+
++ setDelegationToken();
+ URIBuilder uriBuilder = new URIBuilder(remoteAuthorizerServiceUrl);
+ uriBuilder.setPath("/" + CHECK_AUTHORIZATION_OP);
+ uriBuilder.addParameter(WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME,
+ wasbAbsolutePath);
+ uriBuilder.addParameter(ACCESS_OPERATION_QUERY_PARAM_NAME,
+ accessType);
+ if (isSecurityEnabled && StringUtils.isNotEmpty(delegationToken)) {
+ uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME,
+ delegationToken);
+ }
+
+ String responseBody = null;
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ UserGroupInformation connectUgi = ugi.getRealUser();
+ if (connectUgi == null) {
+ connectUgi = ugi;
+ } else {
+ uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName());
+ }
- if (isSecurityEnabled && !connectUgi.hasKerberosCredentials()) {
- connectUgi = UserGroupInformation.getLoginUser();
- }
- connectUgi.checkTGTAndReloginFromKeytab();
+
+ try {
+ responseBody = connectUgi
+ .doAs(new PrivilegedExceptionAction<String>() {
+ @Override
+ public String run() throws Exception {
+ AuthenticatedURL.Token token = null;
+ HttpGet httpGet = new HttpGet(uriBuilder.build());
+ if (isKerberosSupportEnabled && UserGroupInformation
+ .isSecurityEnabled() && (delegationToken == null
+ || delegationToken.isEmpty())) {
+ token = new AuthenticatedURL.Token();
+ final Authenticator kerberosAuthenticator = new
KerberosDelegationTokenAuthenticator();
+ try {
+ kerberosAuthenticator
+ .authenticate(uriBuilder.build().toURL(), token);
+ Validate.isTrue(token.isSet(),
+ "Authenticated Token is NOT present. The request
cannot proceed.");
+ } catch (AuthenticationException e){
+ throw new IOException("Authentication failed in check
authorization", e);
+ }
+ if (token != null) {
+ httpGet.setHeader("Cookie",
+ AuthenticatedURL.AUTH_COOKIE + "=" + token);
}
- return remoteCallHelper.makeRemoteGetRequest(httpGet);
}
- });
- } catch (InterruptedException e) {
- LOG.error("Error in check authorization", e);
- throw new WasbAuthorizationException("Error in check authorize", e);
- }
-
- ObjectMapper objectMapper = new ObjectMapper();
- RemoteAuthorizerResponse authorizerResponse =
- objectMapper
- .readValue(responseBody, RemoteAuthorizerResponse.class);
-
- if (authorizerResponse == null) {
- throw new WasbAuthorizationException(
- "RemoteAuthorizerResponse object null from remote call");
- } else if (authorizerResponse.getResponseCode()
- == REMOTE_CALL_SUCCESS_CODE) {
- return authorizerResponse.getAuthorizationResult();
- } else {
- throw new WasbAuthorizationException("Remote authorization"
- + " service encountered an error "
- + authorizerResponse.getResponseMessage());
- }
- } catch (URISyntaxException | WasbRemoteCallException
- | JsonParseException | JsonMappingException ex) {
- throw new WasbAuthorizationException(ex);
+ return remoteCallHelper.makeRemoteGetRequest(httpGet);
+ }
+ });
+ } catch (InterruptedException e) {
+ LOG.error("Error in check authorization", e);
+ throw new WasbAuthorizationException("Error in check authorize", e);
}
+
+ ObjectMapper objectMapper = new ObjectMapper();
+ RemoteAuthorizerResponse authorizerResponse =
+ objectMapper
+ .readValue(responseBody, RemoteAuthorizerResponse.class);
+
+ if (authorizerResponse == null) {
+ throw new WasbAuthorizationException(
+ "RemoteAuthorizerResponse object null from remote call");
+ } else if (authorizerResponse.getResponseCode()
+ == REMOTE_CALL_SUCCESS_CODE) {
+ return authorizerResponse.getAuthorizationResult();
+ } else {
+ throw new WasbAuthorizationException("Remote authorization"
+ + " service encountered an error "
+ + authorizerResponse.getResponseMessage());
+ }
+ } catch (URISyntaxException | WasbRemoteCallException
+ | JsonParseException | JsonMappingException ex) {
+ throw new WasbAuthorizationException(ex);
+ }
}
+
+ private void setDelegationToken() throws IOException {
+ this.delegationToken = SecurityUtils.getDelegationTokenFromCredentials();
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86414507/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
----------------------------------------------------------------------
diff --cc
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index 6269f21,1b2bca3..fbb56b0
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@@ -1206,11 -1206,11 +1206,11 @@@ public class TestAMRMClient
// Should receive atleast 1 token
Assert.assertTrue(receivedNMTokens.size() > 0
&& receivedNMTokens.size() <= nodeCount);
-
+
assertEquals(allocatedContainerCount, containersRequestedAny);
- assertEquals(2, amClient.release.size());
+ assertEquals(2, releases.size());
assertEquals(0, amClient.ask.size());
-
+
// need to tell the AMRMClient that we dont need these resources anymore
amClient.removeContainerRequest(
new ContainerRequest(capability, nodes, racks, priority));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
