Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2628d1b0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2628d1b0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2628d1b0 Branch: refs/heads/HDFS-7240 Commit: 2628d1b06aa0b1bec2a5bc08105388ee268f6ad2 Parents: 0fb394b c904d60 Author: Anu Engineer <[email protected]> Authored: Mon Oct 23 17:36:16 2017 -0700 Committer: Anu Engineer <[email protected]> Committed: Mon Oct 23 17:36:16 2017 -0700 ---------------------------------------------------------------------- dev-support/docker/Dockerfile | 161 ++--- dev-support/docker/hadoop_env_checks.sh | 15 +- hadoop-client-modules/hadoop-client/pom.xml | 60 ++ .../org/apache/hadoop/conf/Configuration.java | 106 ++-- .../crypto/key/kms/KMSClientProvider.java | 8 +- .../fs/CommonConfigurationKeysPublic.java | 9 + .../org/apache/hadoop/fs/viewfs/ConfigUtil.java | 68 ++- .../org/apache/hadoop/fs/viewfs/Constants.java | 16 +- .../org/apache/hadoop/fs/viewfs/InodeTree.java | 358 ++++++++++-- .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 13 +- .../org/apache/hadoop/fs/viewfs/ViewFs.java | 14 +- .../apache/hadoop/ha/ZKFailoverController.java | 48 +- .../io/erasurecode/ErasureCodeConstants.java | 1 + .../io/erasurecode/coder/ErasureCodingStep.java | 5 +- .../erasurecode/coder/ErasureDecodingStep.java | 5 +- .../erasurecode/coder/ErasureEncodingStep.java | 5 +- .../coder/HHXORErasureDecodingStep.java | 12 +- .../coder/HHXORErasureEncodingStep.java | 10 +- .../io/erasurecode/coder/util/HHUtil.java | 4 +- .../rawcoder/AbstractNativeRawDecoder.java | 14 +- .../rawcoder/AbstractNativeRawEncoder.java | 14 +- .../rawcoder/NativeRSRawDecoder.java | 11 +- .../rawcoder/NativeRSRawEncoder.java | 11 +- .../rawcoder/NativeXORRawDecoder.java | 14 +- .../rawcoder/NativeXORRawEncoder.java | 9 +- .../rawcoder/RSLegacyRawDecoder.java | 6 +- .../erasurecode/rawcoder/RawErasureDecoder.java | 17 +- .../erasurecode/rawcoder/RawErasureEncoder.java | 16 +- .../main/java/org/apache/hadoop/ipc/Client.java | 5 +- .../hadoop/metrics2/source/JvmMetrics.java | 16 + .../hadoop/util/concurrent/ExecutorHelper.java | 10 +- .../apache/hadoop/io/erasurecode/jni_common.c | 5 +- .../hadoop/io/erasurecode/jni_rs_decoder.c | 9 +- .../hadoop/io/erasurecode/jni_rs_encoder.c | 9 +- .../hadoop/io/erasurecode/jni_xor_decoder.c | 9 +- .../hadoop/io/erasurecode/jni_xor_encoder.c | 9 +- .../src/main/resources/core-default.xml | 21 + .../src/site/markdown/GroupsMapping.md | 5 +- .../apache/hadoop/conf/TestConfiguration.java | 15 + .../hadoop/fs/contract/ContractTestUtils.java | 21 + .../fs/viewfs/ViewFileSystemBaseTest.java | 4 +- .../erasurecode/coder/TestErasureCoderBase.java | 18 +- .../coder/TestHHErasureCoderBase.java | 10 +- .../rawcoder/RawErasureCoderBenchmark.java | 9 +- .../erasurecode/rawcoder/TestDummyRawCoder.java | 15 +- .../rawcoder/TestNativeRSRawCoder.java | 6 + .../rawcoder/TestNativeXORRawCoder.java | 7 + .../erasurecode/rawcoder/TestRawCoderBase.java | 61 +- .../hadoop/crypto/key/kms/server/KMSACLs.java | 4 + .../crypto/key/kms/server/KMSConfiguration.java | 15 + .../crypto/key/kms/server/KMSWebServer.java | 24 + .../hadoop/crypto/key/kms/server/TestKMS.java | 44 +- .../hadoop/hdfs/DFSStripedOutputStream.java | 51 +- .../org/apache/hadoop/hdfs/DataStreamer.java | 31 +- .../hadoop/hdfs/DistributedFileSystem.java | 3 +- .../apache/hadoop/hdfs/ExceptionLastSeen.java | 75 +++ .../hadoop/hdfs/PositionStripeReader.java | 3 +- .../hadoop/hdfs/StatefulStripeReader.java | 3 +- .../org/apache/hadoop/hdfs/StripeReader.java | 7 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 16 + .../hadoop/fs/http/client/HttpFSFileSystem.java | 18 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +- .../hdfs/qjournal/client/AsyncLogger.java | 2 +- .../hdfs/qjournal/client/IPCLoggerChannel.java | 43 +- .../qjournal/client/QuorumJournalManager.java | 35 +- .../qjournal/protocol/QJournalProtocol.java | 40 +- .../hdfs/qjournal/protocol/RequestInfo.java | 11 +- .../QJournalProtocolServerSideTranslatorPB.java | 27 +- .../QJournalProtocolTranslatorPB.java | 178 ++++-- .../hadoop/hdfs/qjournal/server/Journal.java | 15 +- .../hdfs/qjournal/server/JournalNode.java | 74 ++- .../qjournal/server/JournalNodeRpcServer.java | 71 ++- .../hdfs/qjournal/server/JournalNodeSyncer.java | 24 +- .../server/blockmanagement/BlockManager.java | 22 +- .../server/datanode/BlockRecoveryWorker.java | 14 +- .../StripedBlockChecksumReconstructor.java | 2 +- .../erasurecode/StripedBlockReconstructor.java | 2 +- .../namenode/ErasureCodingPolicyManager.java | 20 +- .../server/namenode/FSDirEncryptionZoneOp.java | 8 +- .../hadoop/hdfs/server/namenode/FSEditLog.java | 16 +- .../hdfs/server/namenode/FSNamesystem.java | 5 + .../server/namenode/FSPermissionChecker.java | 12 +- .../hdfs/server/namenode/LeaseManager.java | 57 +- .../server/namenode/ReencryptionHandler.java | 106 ++-- .../server/namenode/ReencryptionUpdater.java | 29 +- .../server/namenode/top/metrics/TopMetrics.java | 30 +- .../server/protocol/InterDatanodeProtocol.java | 7 +- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 4 +- .../hdfs/tools/DFSZKFailoverController.java | 3 +- .../src/main/proto/QJournalProtocol.proto | 12 + .../src/main/resources/hdfs-default.xml | 2 +- .../src/site/markdown/HDFSErasureCoding.md | 3 +- .../hadoop-hdfs/src/site/markdown/ViewFs.md | 44 +- .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 5 + .../viewfs/TestViewFileSystemLinkFallback.java | 264 +++++++++ .../TestViewFileSystemLinkMergeSlash.java | 234 ++++++++ .../apache/hadoop/hdfs/StripedFileTestUtil.java | 6 +- .../TestDFSStripedOutputStreamWithFailure.java | 116 +++- ...tputStreamWithFailureWithRandomECPolicy.java | 14 +- .../apache/hadoop/hdfs/TestDatanodeDeath.java | 3 +- .../TestDistributedFileSystemWithECFile.java | 37 +- ...dFileSystemWithECFileWithRandomECPolicy.java | 49 ++ .../hadoop/hdfs/TestErasureCodingPolicies.java | 27 + .../org/apache/hadoop/hdfs/TestFileAppend3.java | 3 +- .../hadoop/hdfs/TestSnapshotCommands.java | 22 +- .../TestTrashWithSecureEncryptionZones.java | 1 - .../qjournal/client/TestEpochsAreUnique.java | 4 +- .../hdfs/qjournal/client/TestQJMWithFaults.java | 12 +- .../client/TestQuorumJournalManager.java | 5 +- .../hdfs/qjournal/server/TestJournal.java | 8 +- .../hdfs/qjournal/server/TestJournalNode.java | 97 +++ .../qjournal/server/TestJournalNodeSync.java | 7 + .../blockmanagement/TestBlockManager.java | 111 ++++ .../hdfs/server/federation/MockResolver.java | 5 +- .../hdfs/server/namenode/TestDeadDatanode.java | 3 +- .../namenode/TestINodeAttributeProvider.java | 60 +- .../namenode/TestQuotaWithStripedBlocks.java | 40 +- ...uotaWithStripedBlocksWithRandomECPolicy.java | 50 ++ .../hdfs/server/namenode/TestReencryption.java | 59 +- .../namenode/ha/TestFailureToReadEdits.java | 25 +- .../server/namenode/metrics/TestTopMetrics.java | 11 +- .../src/test/resources/testHDFSConf.xml | 4 +- .../mapreduce/v2/app/job/impl/JobImpl.java | 16 + .../v2/app/job/impl/TaskAttemptImpl.java | 52 +- .../mapreduce/v2/util/LocalResourceBuilder.java | 169 ++++++ .../apache/hadoop/mapreduce/v2/util/MRApps.java | 137 +---- .../hadoop/mapreduce/v2/util/MRWebAppUtil.java | 13 +- .../TestLocalDistributedCacheManager.java | 9 + .../hadoop/mapreduce/v2/util/TestMRApps.java | 17 +- .../hadoop-mapreduce-client-core/pom.xml | 6 + .../org/apache/hadoop/mapred/RecordReader.java | 6 +- .../java/org/apache/hadoop/mapreduce/Job.java | 226 +++++++ .../hadoop/mapreduce/JobResourceUploader.java | 416 +++++++++++-- .../apache/hadoop/mapreduce/MRJobConfig.java | 71 +++ .../hadoop/mapreduce/SharedCacheConfig.java | 102 ++++ .../src/main/resources/mapred-default.xml | 11 + .../src/site/markdown/SharedCacheSupport.md | 100 ++++ .../mapreduce/TestJobResourceUploader.java | 76 ++- .../TestJobResourceUploaderWithSharedCache.java | 365 ++++++++++++ .../org/apache/hadoop/mapred/YARNRunner.java | 54 +- .../hadoop/mapred/TestLocalJobSubmission.java | 52 ++ .../apache/hadoop/mapreduce/v2/TestMRJobs.java | 59 ++ hadoop-project/pom.xml | 1 + hadoop-project/src/site/site.xml | 1 + .../hadoop/fs/azure/NativeAzureFileSystem.java | 130 ++++- .../hadoop/fs/azure/security/Constants.java | 1 + .../TestNativeAzureFileSystemAuthorization.java | 584 ++++++++++++++++--- .../java/org/apache/hadoop/tools/DistCp.java | 10 +- .../hadoop-sls/src/main/bin/rumen2sls.sh | 3 + hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 3 + .../org/apache/hadoop/yarn/sls/SLSRunner.java | 1 + .../sls/scheduler/CapacitySchedulerMetrics.java | 5 - .../sls/scheduler/FairSchedulerMetrics.java | 17 +- .../sls/scheduler/FifoSchedulerMetrics.java | 5 +- .../sls/scheduler/SLSCapacityScheduler.java | 19 - .../yarn/sls/scheduler/SLSFairScheduler.java | 19 - .../yarn/sls/scheduler/SchedulerMetrics.java | 66 +-- .../hadoop/yarn/api/records/Resource.java | 14 + .../yarn/api/records/ResourceInformation.java | 10 + .../yarn/api/records/ResourceTypeInfo.java | 7 +- .../api/records/impl/LightWeightResource.java | 4 +- .../hadoop/yarn/conf/YarnConfiguration.java | 33 ++ .../yarn/util/resource/ResourceUtils.java | 96 +-- .../hadoop/yarn/api/records/TestResource.java | 43 ++ .../distributedshell/TestDistributedShell.java | 1 - .../yarn/client/api/impl/TestAMRMProxy.java | 4 + .../api/records/impl/pb/ResourcePBImpl.java | 4 +- .../tfile/LogAggregationTFileController.java | 9 +- .../org/apache/hadoop/yarn/util/FSDownload.java | 17 +- .../hadoop/yarn/util/resource/Resources.java | 11 +- .../org/apache/hadoop/yarn/webapp/WebApps.java | 22 + .../src/main/resources/yarn-default.xml | 42 +- .../hadoop/yarn/api/TestResourcePBImpl.java | 27 + .../yarn/util/resource/TestResourceUtils.java | 17 + .../policies/FederationPolicyUtils.java | 41 +- .../LocalityMulticastAMRMProxyPolicy.java | 103 +++- .../router/WeightedRandomRouterPolicy.java | 33 +- .../BaseContainerTokenSecretManager.java | 8 +- .../policies/TestFederationPolicyUtils.java | 58 ++ .../TestLocalityMulticastAMRMProxyPolicy.java | 110 +++- .../server/nodemanager/ContainerExecutor.java | 3 +- .../hadoop/yarn/server/nodemanager/Context.java | 3 + .../nodemanager/DefaultContainerExecutor.java | 2 +- .../nodemanager/LinuxContainerExecutor.java | 10 +- .../yarn/server/nodemanager/NodeManager.java | 92 ++- .../nodemanager/NodeStatusUpdaterImpl.java | 38 +- .../containermanager/ContainerManagerImpl.java | 9 + .../container/ContainerEventType.java | 4 +- .../container/ContainerImpl.java | 43 +- .../launcher/ContainersLauncher.java | 2 - .../launcher/RecoverPausedContainerLaunch.java | 38 +- .../launcher/RecoveredContainerLaunch.java | 2 +- .../linux/privileged/PrivilegedOperation.java | 1 + .../linux/resources/ResourceHandlerChain.java | 4 +- .../linux/resources/ResourceHandlerModule.java | 42 +- .../resources/gpu/GpuResourceAllocator.java | 242 ++++++++ .../resources/gpu/GpuResourceHandlerImpl.java | 153 +++++ .../localizer/ResourceLocalizationService.java | 11 +- .../NodeResourceUpdaterPlugin.java | 52 ++ .../resourceplugin/ResourcePlugin.java | 83 +++ .../resourceplugin/ResourcePluginManager.java | 106 ++++ .../resourceplugin/gpu/GpuDiscoverer.java | 254 ++++++++ .../gpu/GpuNodeResourceUpdateHandler.java | 66 +++ .../resourceplugin/gpu/GpuResourcePlugin.java | 61 ++ .../scheduler/ContainerScheduler.java | 31 + .../scheduler/ContainerSchedulerEventType.java | 3 +- .../recovery/NMLeveldbStateStoreService.java | 62 +- .../recovery/NMNullStateStoreService.java | 6 +- .../recovery/NMStateStoreService.java | 10 +- .../webapp/dao/gpu/GpuDeviceInformation.java | 72 +++ .../dao/gpu/GpuDeviceInformationParser.java | 87 +++ .../webapp/dao/gpu/PerGpuDeviceInformation.java | 165 ++++++ .../webapp/dao/gpu/PerGpuMemoryUsage.java | 58 ++ .../webapp/dao/gpu/PerGpuTemperature.java | 80 +++ .../webapp/dao/gpu/PerGpuUtilizations.java | 50 ++ .../impl/container-executor.c | 41 +- .../impl/modules/gpu/gpu-module.c | 2 +- .../container-executor/impl/utils/docker-util.c | 5 +- .../test/modules/gpu/test-gpu-module.cc | 13 + .../test/test-container-executor.c | 8 + .../test/utils/test_docker_util.cc | 103 ++-- .../server/nodemanager/NodeManagerTestBase.java | 164 ++++++ .../TestDefaultContainerExecutor.java | 4 +- .../nodemanager/TestLinuxContainerExecutor.java | 2 +- .../TestLinuxContainerExecutorWithMocks.java | 2 +- .../server/nodemanager/TestNodeManager.java | 2 +- .../nodemanager/TestNodeStatusUpdater.java | 100 +--- .../amrmproxy/BaseAMRMProxyTest.java | 46 +- .../TestContainerManagerRecovery.java | 4 + .../resources/TestResourceHandlerModule.java | 8 +- .../resources/gpu/TestGpuResourceHandler.java | 382 ++++++++++++ .../TestContainersMonitorResourceChange.java | 2 +- .../TestResourcePluginManager.java | 261 +++++++++ .../resourceplugin/gpu/TestGpuDiscoverer.java | 123 ++++ .../TestContainerSchedulerQueuing.java | 30 +- .../recovery/NMMemoryStateStoreService.java | 18 +- .../TestNMLeveldbStateStoreService.java | 16 +- .../dao/gpu/TestGpuDeviceInformationParser.java | 50 ++ .../test/resources/nvidia-smi-sample-xml-output | 547 +++++++++++++++++ .../server/resourcemanager/ClientRMService.java | 16 + .../server/resourcemanager/RMAuditLogger.java | 172 +++++- .../reservation/FairReservationSystem.java | 13 + .../resourcemanager/resource/ResourceType.java | 28 - .../scheduler/fair/FSAppAttempt.java | 19 + .../scheduler/fair/FSLeafQueue.java | 19 +- .../scheduler/fair/FSParentQueue.java | 14 - .../resourcemanager/scheduler/fair/FSQueue.java | 34 ++ .../fair/FairSchedulerConfiguration.java | 8 +- .../ParameterizedSchedulerTestBase.java | 3 +- .../server/resourcemanager/TestAppManager.java | 2 - .../resourcemanager/TestApplicationACLs.java | 90 ++- .../resourcemanager/TestRMAuditLogger.java | 47 +- .../resourcemanager/TestSignalContainer.java | 8 + .../recovery/TestZKRMStateStore.java | 5 + .../scheduler/fair/TestFairScheduler.java | 39 +- .../webapp/TestRMWebServicesReservation.java | 28 +- .../router/webapp/RouterWebServiceUtil.java | 36 +- .../router/webapp/TestRouterWebServiceUtil.java | 17 +- .../server/ContainerTokenIdentifierForTest.java | 7 + .../webproxy/amfilter/AmFilterInitializer.java | 16 +- .../amfilter/TestAmFilterInitializer.java | 8 +- .../hadoop-yarn/hadoop-yarn-ui/pom.xml | 2 +- .../hadoop-yarn-ui/src/main/webapp/.bowerrc | 5 +- 263 files changed, 10054 insertions(+), 1786 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2628d1b0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2628d1b0/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2628d1b0/hadoop-project/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2628d1b0/hadoop-project/src/site/site.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
