Merge branch 'trunk' into HDFS-7240
Conflicts:
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e55bdefd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e55bdefd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e55bdefd
Branch: refs/heads/HDFS-7240
Commit: e55bdefdab3c8a7f1e51451bb8280e61f2c6f0db
Parents: e6a6b5b 503e73e
Author: Anu Engineer <[email protected]>
Authored: Fri Nov 11 10:49:05 2016 -0800
Committer: Anu Engineer <[email protected]>
Committed: Fri Nov 11 10:49:05 2016 -0800
----------------------------------------------------------------------
.gitignore | 14 +
BUILDING.txt | 23 +-
LICENSE.txt | 546 +++
dev-support/bin/checkcompatibility.py | 345 ++
dev-support/bin/create-release | 2 +-
dev-support/docker/Dockerfile | 9 +
.../resources/assemblies/hadoop-yarn-dist.xml | 7 +
.../main/resources/checkstyle/checkstyle.xml | 2 +-
hadoop-client/pom.xml | 20 +-
.../hadoop-cloud-storage/pom.xml | 127 +
hadoop-cloud-storage-project/pom.xml | 54 +
.../hadoop-auth-examples/pom.xml | 2 +-
.../examples/RequestLoggerFilter.java | 12 +
hadoop-common-project/hadoop-auth/pom.xml | 13 +-
.../server/KerberosAuthenticationHandler.java | 7 +-
.../client/AuthenticatorTestCase.java | 29 +-
hadoop-common-project/hadoop-common/pom.xml | 45 +-
.../hadoop-common/src/CMakeLists.txt | 4 +-
.../hadoop-common/src/main/bin/hadoop | 2 +-
.../hadoop-common/src/main/conf/hadoop-env.sh | 6 -
.../src/main/conf/hadoop-metrics2.properties | 22 +-
.../org/apache/hadoop/conf/ConfServlet.java | 8 +-
.../crypto/key/kms/KMSClientProvider.java | 56 +-
.../crypto/key/kms/KMSDelegationToken.java | 52 +
.../fs/CommonConfigurationKeysPublic.java | 2 +
.../apache/hadoop/fs/FSExceptionMessages.java | 2 +
.../java/org/apache/hadoop/fs/FileSystem.java | 9 +-
.../java/org/apache/hadoop/fs/FileUtil.java | 8 +-
.../org/apache/hadoop/fs/HarFileSystem.java | 3 +
.../java/org/apache/hadoop/fs/shell/Count.java | 9 +-
.../org/apache/hadoop/fs/viewfs/InodeTree.java | 6 +
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 15 +-
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 8 +-
.../hadoop/http/AdminAuthorizedServlet.java | 13 +-
.../org/apache/hadoop/http/HttpRequestLog.java | 4 +-
.../org/apache/hadoop/http/HttpServer2.java | 337 +-
.../java/org/apache/hadoop/http/JettyUtils.java | 35 +
.../rawcoder/NativeXORRawDecoder.java | 59 +
.../rawcoder/NativeXORRawEncoder.java | 60 +
.../NativeXORRawErasureCoderFactory.java | 39 +
.../hadoop/ipc/metrics/RpcDetailedMetrics.java | 4 +-
.../org/apache/hadoop/jmx/JMXJsonServlet.java | 8 +-
.../java/org/apache/hadoop/log/LogLevel.java | 11 +-
.../metrics2/lib/MutableMetricsFactory.java | 5 +
.../hadoop/metrics2/lib/MutableRates.java | 6 +
.../lib/MutableRatesWithAggregation.java | 148 +
.../apache/hadoop/metrics2/lib/MutableStat.java | 4 +
.../apache/hadoop/metrics2/util/SampleStat.java | 19 +-
.../apache/hadoop/security/SaslInputStream.java | 5 +-
.../apache/hadoop/security/SaslRpcClient.java | 3 +
.../hadoop/security/UserGroupInformation.java | 84 +-
.../ssl/SslSelectChannelConnectorSecure.java | 58 -
.../AbstractDelegationTokenSecretManager.java | 69 +-
.../DelegationTokenAuthenticationHandler.java | 32 +-
.../org/apache/hadoop/util/DiskChecker.java | 159 +-
.../hadoop/util/GenericOptionsParser.java | 37 +-
.../apache/hadoop/util/InstrumentedLock.java | 197 +
.../hadoop/util/InstrumentedReadLock.java | 92 +
.../hadoop/util/InstrumentedReadWriteLock.java | 58 +
.../hadoop/util/InstrumentedWriteLock.java | 54 +
.../apache/hadoop/util/LimitInputStream.java | 3 +
.../main/java/org/apache/hadoop/util/Shell.java | 8 +-
.../main/java/org/apache/hadoop/util/Time.java | 18 +
.../src/main/native/native.vcxproj | 6 +-
.../hadoop/io/erasurecode/jni_xor_decoder.c | 80 +
.../hadoop/io/erasurecode/jni_xor_encoder.c | 82 +
...apache.hadoop.security.token.TokenIdentifier | 14 +
.../src/main/resources/core-default.xml | 164 +-
.../src/site/markdown/DeprecatedProperties.md | 24 -
.../src/site/markdown/FileSystemShell.md | 2 +-
.../site/markdown/filesystem/introduction.md | 15 +
.../org/apache/hadoop/conf/TestConfServlet.java | 2 +-
.../apache/hadoop/conf/TestConfigRedactor.java | 2 +
.../conf/TestConfigurationFieldsBase.java | 47 +
.../hadoop/fs/FSMainOperationsBaseTest.java | 4 +-
.../org/apache/hadoop/fs/TestDefaultUri.java | 115 +
.../org/apache/hadoop/fs/TestFsShellCopy.java | 51 +
.../fs/contract/AbstractContractCreateTest.java | 18 +-
.../hadoop/fs/contract/ContractOptions.java | 18 +
.../hadoop/fs/contract/ContractTestUtils.java | 6 +
.../org/apache/hadoop/fs/shell/TestCount.java | 9 +-
.../TestFSMainOperationsLocalFileSystem.java | 4 +-
.../fs/viewfs/ViewFileSystemBaseTest.java | 41 +-
.../fs/viewfs/ViewFileSystemTestSetup.java | 10 +-
.../hadoop/fs/viewfs/ViewFsTestSetup.java | 10 +-
.../http/TestAuthenticationSessionCookie.java | 11 +-
.../apache/hadoop/http/TestHttpRequestLog.java | 4 +-
.../org/apache/hadoop/http/TestHttpServer.java | 39 +-
.../apache/hadoop/http/TestServletFilter.java | 7 +-
.../hadoop/http/resource/JerseyResource.java | 5 +-
.../rawcoder/TestNativeXORRawCoder.java | 36 +
.../erasurecode/rawcoder/TestXORRawCoder.java | 38 +-
.../rawcoder/TestXORRawCoderBase.java | 59 +
.../rawcoder/TestXORRawCoderInteroperable1.java | 36 +
.../rawcoder/TestXORRawCoderInteroperable2.java | 37 +
.../org/apache/hadoop/io/file/tfile/Timer.java | 52 +-
.../hadoop/metrics2/lib/TestMutableMetrics.java | 165 +-
.../hadoop/security/TestUGIWithMiniKdc.java | 144 +
.../security/TestUserGroupInformation.java | 93 +
...tionTokenAuthenticationHandlerWithMocks.java | 50 +
.../delegation/web/TestWebDelegationToken.java | 64 +-
.../org/apache/hadoop/util/TestDiskChecker.java | 178 +-
.../hadoop/util/TestInstrumentedLock.java | 162 +
.../util/TestInstrumentedReadWriteLock.java | 234 +
.../java/org/apache/hadoop/util/TestTime.java | 50 +
.../src/test/resources/contract/ftp.xml | 7 +-
.../dev-support/findbugsExcludeFile.xml | 2 +-
hadoop-common-project/hadoop-kms/pom.xml | 20 +-
.../hadoop/crypto/key/kms/server/KMS.java | 21 +-
.../key/kms/server/KMSAuthenticationFilter.java | 16 +-
.../crypto/key/kms/server/KMSJSONWriter.java | 3 +-
.../hadoop-kms/src/site/markdown/index.md.vm | 2 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 63 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 19 +-
hadoop-common-project/hadoop-nfs/pom.xml | 2 +-
hadoop-dist/pom.xml | 45 +
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 6 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 30 +-
.../hadoop/hdfs/DFSOpsCountStatistics.java | 1 +
.../hadoop/hdfs/DFSStripedOutputStream.java | 4 +-
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 50 +
.../hadoop/hdfs/protocol/DatanodeInfo.java | 217 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 22 +-
.../server/datanode/DiskBalancerWorkItem.java | 4 +-
.../server/datanode/DiskBalancerWorkStatus.java | 13 +-
.../hdfs/util/CombinedHostsFileReader.java | 8 +-
.../hdfs/util/CombinedHostsFileWriter.java | 2 +-
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 46 +-
.../hadoop/hdfs/web/URLConnectionFactory.java | 2 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 28 +-
...onfRefreshTokenBasedAccessTokenProvider.java | 4 +-
.../CredentialBasedAccessTokenProvider.java | 4 +-
.../hadoop/hdfs/web/resources/GetOpParam.java | 1 +
.../hdfs/web/TestURLConnectionFactory.java | 17 +
.../hadoop/hdfs/web/TestWebHDFSOAuth2.java | 2 +-
...ClientCredentialTimeBasedTokenRefresher.java | 2 +-
...TestRefreshTokenTimeBasedTokenRefresher.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 26 +-
.../hadoop/fs/http/client/HttpFSFileSystem.java | 31 +-
.../hadoop/fs/http/server/FSOperations.java | 23 +
.../http/server/HttpFSParametersProvider.java | 1 +
.../hadoop/fs/http/server/HttpFSServer.java | 19 +-
.../org/apache/hadoop/lib/server/Server.java | 15 +-
.../apache/hadoop/lib/wsrs/JSONMapProvider.java | 3 +-
.../apache/hadoop/lib/wsrs/JSONProvider.java | 3 +-
.../src/site/markdown/index.md | 2 +
.../fs/http/client/BaseTestHttpFSWith.java | 191 +-
.../TestHttpFSFWithSWebhdfsFileSystem.java | 6 +-
.../hadoop/fs/http/server/TestHttpFSServer.java | 76 +-
.../fs/http/server/TestHttpFSServerNoACLs.java | 6 +-
.../http/server/TestHttpFSServerNoXAttrs.java | 6 +-
.../fs/http/server/TestHttpFSWithKerberos.java | 6 +-
.../org/apache/hadoop/test/TestHFSTestCase.java | 8 +-
.../org/apache/hadoop/test/TestHTestCase.java | 8 +-
.../org/apache/hadoop/test/TestJettyHelper.java | 56 +-
.../src/test/resources/default-log4j.properties | 8 +-
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 18 +-
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 41 +-
.../hadoop-hdfs/src/main/bin/hdfs | 2 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +
.../apache/hadoop/hdfs/InstrumentedLock.java | 185 -
.../DatanodeProtocolServerSideTranslatorPB.java | 4 +-
.../NamenodeProtocolServerSideTranslatorPB.java | 6 +-
.../hdfs/qjournal/server/JournalNode.java | 2 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 5 +-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 49 +-
.../server/blockmanagement/BlockIdManager.java | 1 +
.../blockmanagement/BlockManagerSafeMode.java | 19 +-
.../BlockPlacementPolicyDefault.java | 8 +-
.../CacheReplicationMonitor.java | 14 +-
.../server/blockmanagement/DatanodeManager.java | 16 +-
.../hadoop/hdfs/server/common/Storage.java | 82 +-
.../server/datanode/BlockPoolSliceStorage.java | 48 +-
.../hdfs/server/datanode/BlockSender.java | 7 +
.../hadoop/hdfs/server/datanode/DataNode.java | 25 +-
.../hdfs/server/datanode/DataStorage.java | 77 +-
.../hdfs/server/datanode/DirectoryScanner.java | 14 +-
.../hdfs/server/datanode/LocalReplica.java | 8 +-
.../server/datanode/ReportBadBlockAction.java | 4 +-
.../hdfs/server/datanode/StorageLocation.java | 108 +-
.../server/datanode/checker/AsyncChecker.java | 63 +
.../hdfs/server/datanode/checker/Checkable.java | 49 +
.../datanode/checker/ThrottledAsyncChecker.java | 224 +
.../server/datanode/checker/package-info.java | 26 +
.../erasurecode/ErasureCodingWorker.java | 23 +-
.../StripedBlockChecksumReconstructor.java | 1 +
.../erasurecode/StripedBlockReconstructor.java | 3 +
.../erasurecode/StripedBlockWriter.java | 4 +-
.../server/datanode/fsdataset/FsDatasetSpi.java | 14 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 54 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 49 +-
.../datanode/metrics/DataNodeMetrics.java | 13 +-
.../diskbalancer/DiskBalancerException.java | 1 +
.../server/diskbalancer/command/Command.java | 23 +-
.../diskbalancer/command/PlanCommand.java | 15 +-
.../connectors/JsonNodeConnector.java | 4 +-
.../datamodel/DiskBalancerCluster.java | 8 +-
.../datamodel/DiskBalancerVolume.java | 8 +-
.../datamodel/DiskBalancerVolumeSet.java | 6 +-
.../server/diskbalancer/planner/NodePlan.java | 6 +-
.../hdfs/server/namenode/CacheManager.java | 8 +-
.../server/namenode/EncryptionZoneManager.java | 3 +-
.../hadoop/hdfs/server/namenode/FSDirAclOp.java | 28 +-
.../hdfs/server/namenode/FSDirAppendOp.java | 3 +-
.../hdfs/server/namenode/FSDirAttrOp.java | 26 +-
.../hdfs/server/namenode/FSDirConcatOp.java | 8 +-
.../hdfs/server/namenode/FSDirDeleteOp.java | 15 +-
.../server/namenode/FSDirEncryptionZoneOp.java | 5 +-
.../server/namenode/FSDirErasureCodingOp.java | 5 +-
.../hdfs/server/namenode/FSDirMkdirOp.java | 18 +-
.../hdfs/server/namenode/FSDirRenameOp.java | 24 +-
.../hdfs/server/namenode/FSDirSnapshotOp.java | 17 +-
.../server/namenode/FSDirStatAndListingOp.java | 49 +-
.../hdfs/server/namenode/FSDirSymlinkOp.java | 3 +-
.../hdfs/server/namenode/FSDirTruncateOp.java | 9 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 3 +-
.../hdfs/server/namenode/FSDirXAttrOp.java | 12 +-
.../hdfs/server/namenode/FSDirectory.java | 191 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 54 +-
.../hdfs/server/namenode/FSImageFormat.java | 17 +-
.../hdfs/server/namenode/FSNamesystem.java | 33 +-
.../server/namenode/FSPermissionChecker.java | 165 +-
.../hdfs/server/namenode/INodesInPath.java | 90 +-
.../hadoop/hdfs/server/namenode/NNStorage.java | 4 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 1 -
.../hdfs/server/namenode/NameNodeRpcServer.java | 1 -
.../hdfs/server/namenode/NamenodeFsck.java | 15 +-
.../server/namenode/StartupProgressServlet.java | 6 +-
.../hdfs/server/namenode/TransferFsImage.java | 2 +-
.../namenode/snapshot/SnapshotManager.java | 5 +-
.../web/resources/NamenodeWebHdfsMethods.java | 82 +-
.../protocol/StorageReceivedDeletedBlocks.java | 7 +
.../hadoop/hdfs/tools/DiskBalancerCLI.java | 69 +-
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 2 +-
.../impl/ContainerLocationManagerImpl.java | 8 +-
.../common/impl/ContainerManagerImpl.java | 2 +-
.../src/main/resources/hdfs-default.xml | 39 +-
.../src/site/markdown/HDFSDiskbalancer.md | 1 +
.../src/site/markdown/HDFSErasureCoding.md | 11 +-
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 37 +-
.../fs/viewfs/TestViewFsDefaultValue.java | 12 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 37 +-
.../hadoop/hdfs/TestDFSClientSocketSize.java | 6 +-
.../apache/hadoop/hdfs/TestDatanodeReport.java | 38 +-
.../apache/hadoop/hdfs/TestDecommission.java | 2 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 102 +-
.../apache/hadoop/hdfs/TestExtendedAcls.java | 441 ++
.../org/apache/hadoop/hdfs/TestFileAppend.java | 71 +
.../apache/hadoop/hdfs/TestFileCorruption.java | 7 +-
.../org/apache/hadoop/hdfs/TestFileStatus.java | 4 +-
.../hadoop/hdfs/TestInstrumentedLock.java | 166 -
.../java/org/apache/hadoop/hdfs/TestQuota.java | 2104 ++++----
.../hadoop/hdfs/TestReservedRawPaths.java | 5 +-
.../hdfs/TestSecureEncryptionZoneWithKMS.java | 9 +-
.../client/impl/TestBlockReaderFactory.java | 6 +-
.../sasl/SaslDataTransferTestCase.java | 9 +-
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 4 +-
.../hdfs/qjournal/TestSecureNNWithQJM.java | 11 +-
.../qjournal/server/TestJournalNodeMXBean.java | 2 +-
.../hdfs/server/balancer/TestBalancer.java | 16 +-
.../TestBlockManagerSafeMode.java | 20 +
.../blockmanagement/TestBlockStatsMXBean.java | 2 +-
.../blockmanagement/TestDatanodeManager.java | 37 +
.../TestNameNodePrunesMissingStorages.java | 7 +-
.../TestPendingReconstruction.java | 12 +-
.../TestSequentialBlockGroupId.java | 5 +
.../server/datanode/SimulatedFSDataset.java | 5 -
.../hdfs/server/datanode/TestBlockRecovery.java | 6 +-
.../hdfs/server/datanode/TestBlockScanner.java | 6 +-
.../hdfs/server/datanode/TestDataDirs.java | 3 +-
.../TestDataNodeErasureCodingMetrics.java | 43 +-
.../datanode/TestDataNodeHotSwapVolumes.java | 24 +-
.../server/datanode/TestDataNodeMXBean.java | 6 +-
.../datanode/TestDataNodeVolumeFailure.java | 177 +-
.../TestDataNodeVolumeFailureReporting.java | 37 +-
.../hdfs/server/datanode/TestDataStorage.java | 7 +-
.../server/datanode/TestDirectoryScanner.java | 4 +-
.../hdfs/server/datanode/TestDiskError.java | 3 +-
.../datanode/TestIncrementalBrVariations.java | 4 +-
.../checker/TestThrottledAsyncChecker.java | 276 ++
.../extdataset/ExternalDatasetImpl.java | 5 -
.../fsdataset/impl/FsDatasetImplTestUtils.java | 14 +
.../fsdataset/impl/FsDatasetTestUtil.java | 2 +-
.../impl/TestInterDatanodeProtocol.java | 4 +-
.../diskbalancer/DiskBalancerTestUtil.java | 21 +-
.../diskbalancer/TestDiskBalancerRPC.java | 2 +-
.../command/TestDiskBalancerCommand.java | 178 +-
.../hdfs/server/namenode/FSAclBaseTest.java | 7 +-
.../hdfs/server/namenode/FSXAttrBaseTest.java | 3 +-
.../server/namenode/NNThroughputBenchmark.java | 5 +-
.../hdfs/server/namenode/NameNodeAdapter.java | 5 +-
.../hdfs/server/namenode/TestAuditLogs.java | 9 +
.../hdfs/server/namenode/TestDeadDatanode.java | 3 +-
.../namenode/TestDiskspaceQuotaUpdate.java | 19 +-
.../hdfs/server/namenode/TestFSDirectory.java | 37 +-
.../server/namenode/TestFSNamesystemMBean.java | 2 +-
.../namenode/TestFSPermissionChecker.java | 5 +-
.../hdfs/server/namenode/TestFileTruncate.java | 5 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 12 +-
.../server/namenode/TestGetBlockLocations.java | 5 +-
.../namenode/TestNameNodeConfiguration.java | 40 +
.../server/namenode/TestNameNodeMXBean.java | 4 +-
.../TestNameNodeMetadataConsistency.java | 45 +-
.../TestNameNodeRespectsBindHostKeys.java | 10 +-
.../server/namenode/TestQuotaByStorageType.java | 166 +-
.../server/namenode/TestSnapshotPathINodes.java | 8 +
.../hdfs/server/namenode/TestStartup.java | 55 +-
.../namenode/TestStartupProgressServlet.java | 2 +-
.../server/namenode/TestTransferFsImage.java | 2 +-
.../namenode/snapshot/SnapshotTestHelper.java | 10 +-
.../snapshot/TestSnapshotReplication.java | 3 +-
.../shortcircuit/TestShortCircuitCache.java | 11 +-
.../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 145 +-
.../apache/hadoop/hdfs/web/TestJsonUtil.java | 4 +-
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 43 +-
.../hadoop/hdfs/web/TestWebHDFSForHA.java | 2 +-
.../hadoop/hdfs/web/TestWebHdfsTokens.java | 5 +-
.../apache/hadoop/security/TestPermission.java | 10 +-
.../hadoop/security/TestPermissionSymlinks.java | 7 +-
.../hadoop/test/MiniDFSClusterManager.java | 2 +-
.../hadoop-mapreduce-client-app/pom.xml | 4 +
.../jobhistory/JobHistoryEventHandler.java | 4 +-
.../hadoop/mapreduce/v2/app/JobEndNotifier.java | 37 +-
.../v2/app/rm/RMContainerAllocator.java | 201 +-
.../mapreduce/v2/app/webapp/AMWebServices.java | 49 +-
.../jobhistory/TestJobHistoryEventHandler.java | 5 +-
.../v2/app/rm/TestRMContainerAllocator.java | 218 +-
.../v2/app/webapp/TestAMWebServices.java | 31 +-
.../v2/app/webapp/TestAMWebServicesAttempt.java | 13 +-
.../app/webapp/TestAMWebServicesAttempts.java | 34 +-
.../v2/app/webapp/TestAMWebServicesJobConf.java | 13 +-
.../v2/app/webapp/TestAMWebServicesJobs.java | 64 +-
.../v2/app/webapp/TestAMWebServicesTasks.java | 61 +-
.../hadoop-mapreduce-client-common/pom.xml | 2 -
.../hadoop-mapreduce-client-core/pom.xml | 4 +
.../apache/hadoop/mapred/IFileOutputStream.java | 8 +-
.../java/org/apache/hadoop/mapred/JobConf.java | 11 +-
.../org/apache/hadoop/mapred/QueueManager.java | 8 +-
.../hadoop/mapred/pipes/BinaryProtocol.java | 14 +-
.../hadoop/mapreduce/JobSubmissionFiles.java | 63 +-
.../apache/hadoop/mapreduce/JobSubmitter.java | 17 +-
.../mapreduce/server/jobtracker/JTConfig.java | 81 +-
.../hadoop/mapreduce/task/reduce/Fetcher.java | 36 +
.../hadoop/mapreduce/task/reduce/MapHost.java | 4 -
.../task/reduce/ShuffleSchedulerImpl.java | 43 +-
.../hadoop/mapreduce/util/ConfigUtil.java | 53 -
.../mapreduce/util/JobHistoryEventUtils.java | 9 +-
.../src/main/resources/mapred-default.xml | 13 +
.../mapreduce/TestJobSubmissionFiles.java | 142 +
.../mapreduce/task/reduce/TestFetcher.java | 22 +
.../mapreduce/v2/hs/webapp/HsTasksPage.java | 3 +-
.../mapreduce/v2/hs/webapp/HsWebServices.java | 40 +-
.../src/site/markdown/HistoryServerRest.md | 2 +-
.../mapreduce/v2/hs/TestJobHistoryParsing.java | 2 +
.../v2/hs/webapp/TestHsWebServices.java | 25 +-
.../v2/hs/webapp/TestHsWebServicesAttempts.java | 34 +-
.../v2/hs/webapp/TestHsWebServicesJobConf.java | 13 +-
.../v2/hs/webapp/TestHsWebServicesJobs.java | 67 +-
.../hs/webapp/TestHsWebServicesJobsQuery.java | 76 +-
.../v2/hs/webapp/TestHsWebServicesTasks.java | 61 +-
.../resources/job_1329348432655_0001_conf.xml | 15 -
.../hadoop/ipc/TestMRCJCSocketFactory.java | 5 +-
.../hadoop/mapred/NotificationTestCase.java | 12 +-
.../hadoop/mapred/TestClientRedirect.java | 9 +
.../apache/hadoop/mapred/TestIFileStreams.java | 20 +
.../mapreduce/MiniHadoopClusterManager.java | 2 +-
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 5 +-
.../hadoop/mapreduce/v2/TestRMNMInfo.java | 4 +-
.../hadoop-mapreduce-client-shuffle/pom.xml | 2 -
.../apache/hadoop/mapred/ShuffleHandler.java | 41 +-
.../hadoop/mapred/TestShuffleHandler.java | 42 +-
.../hadoop-mapreduce-client/pom.xml | 4 +-
hadoop-mapreduce-project/pom.xml | 4 +-
hadoop-maven-plugins/pom.xml | 4 +
.../hadoop/maven/plugin/protoc/ProtocMojo.java | 228 +-
.../maven/plugin/protoc/ProtocRunner.java | 283 ++
.../maven/plugin/protoc/ProtocTestMojo.java | 61 +
.../maven/plugin/protoc/package-info.java | 98 +
.../maven/plugin/resourcegz/ResourceGzMojo.java | 131 +
hadoop-project-dist/pom.xml | 2 +-
hadoop-project/pom.xml | 62 +-
hadoop-project/src/site/site.xml | 1 -
.../apache/hadoop/tools/TestHadoopArchives.java | 2 +-
hadoop-tools/hadoop-aws/pom.xml | 7 -
.../fs/s3a/AWSCredentialProviderList.java | 11 +
.../java/org/apache/hadoop/fs/s3a/Listing.java | 5 +-
.../org/apache/hadoop/fs/s3a/S3AFileStatus.java | 33 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 46 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 155 +-
...haredInstanceProfileCredentialsProvider.java | 67 +
.../src/site/markdown/tools/hadoop-aws/index.md | 108 +-
.../fs/contract/s3a/ITestS3AContractDistCp.java | 6 +
.../hadoop/fs/contract/s3a/S3AContract.java | 6 +-
.../hadoop/fs/s3a/AbstractS3ATestBase.java | 26 +-
.../fs/s3a/ITestS3AAWSCredentialsProvider.java | 113 +-
.../fs/s3a/ITestS3ABlockingThreadPool.java | 82 -
.../apache/hadoop/fs/s3a/ITestS3ABlocksize.java | 19 +-
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 59 +-
.../hadoop/fs/s3a/ITestS3AEncryption.java | 9 +-
.../ITestS3AEncryptionAlgorithmPropagation.java | 7 -
.../hadoop/fs/s3a/ITestS3AFailureHandling.java | 11 +-
.../fs/s3a/ITestS3AFileOperationCost.java | 19 +-
.../fs/s3a/ITestS3AFileSystemContract.java | 33 +-
.../fs/s3a/ITestS3ATemporaryCredentials.java | 14 +-
.../apache/hadoop/fs/s3a/S3ATestConstants.java | 27 +-
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 58 +-
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 273 ++
.../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 25 +-
.../s3a/scale/ITestS3ADeleteFilesOneByOne.java | 12 +-
.../fs/s3a/scale/ITestS3ADeleteManyFiles.java | 13 +-
.../s3a/scale/ITestS3ADirectoryPerformance.java | 16 +-
.../scale/ITestS3AHugeFilesClassicOutput.java | 4 +-
.../scale/ITestS3AInputStreamPerformance.java | 3 +-
.../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 139 +-
.../org/apache/hadoop/fs/s3a/yarn/ITestS3A.java | 4 +-
.../fs/s3a/yarn/ITestS3AMiniYarnCluster.java | 50 +-
.../src/test/resources/contract/s3a.xml | 5 +
.../src/test/resources/contract/s3n.xml | 10 +
hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +-
hadoop-tools/hadoop-azure/pom.xml | 7 +-
.../fs/azure/AzureNativeFileSystemStore.java | 2 +-
.../hadoop/fs/azure/BlockBlobAppendStream.java | 6 +-
...stNativeAzureFileSystemOperationsMocked.java | 9 +
.../hadoop/tools/CopyListingFileStatus.java | 129 +-
.../apache/hadoop/tools/mapred/CopyMapper.java | 27 +-
.../tools/mapred/RetriableFileCopyCommand.java | 56 +-
.../hadoop/tools/util/ThrottledInputStream.java | 3 +
.../src/site/markdown/DistCp.md.vm | 3 +-
.../hadoop/tools/TestCopyListingFileStatus.java | 67 +
.../mapred/TestRetriableFileCopyCommand.java | 5 +-
.../hadoop/mapred/gridmix/GridmixTestUtils.java | 5 +-
hadoop-tools/hadoop-openstack/pom.xml | 2 +-
.../swift/http/HttpInputStreamWithRelease.java | 3 +
.../swift/snative/SwiftNativeInputStream.java | 3 +
.../fs/swift/TestFSMainOperationsSwift.java | 6 +
.../src/test/resources/contract/swift.xml | 10 +
hadoop-tools/hadoop-sls/pom.xml | 10 +-
.../src/main/data/2jobs2min-rumen-jh.json | 52 +-
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 9 +-
.../yarn/sls/scheduler/RMNodeWrapper.java | 9 +-
.../apache/hadoop/yarn/sls/web/SLSWebApp.java | 19 +-
.../dev-support/findbugs-exclude.xml | 9 +
.../hadoop-yarn/hadoop-yarn-api/pom.xml | 2 -
.../yarn/api/ApplicationClientProtocol.java | 23 +
.../UpdateApplicationTimeoutsRequest.java | 81 +
.../UpdateApplicationTimeoutsResponse.java | 46 +
.../records/ApplicationResourceUsageReport.java | 43 +-
.../records/ApplicationSubmissionContext.java | 4 +
.../yarn/api/records/ResourceRequest.java | 2 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 32 +-
.../main/proto/applicationclient_protocol.proto | 1 +
.../src/main/proto/yarn_protos.proto | 7 +
.../src/main/proto/yarn_service_protos.proto | 9 +
.../yarn/conf/TestYarnConfigurationFields.java | 15 +-
.../distributedshell/TestDistributedShell.java | 9 +-
.../TestDistributedShellWithNodeLabels.java | 12 +-
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 2 -
.../yarn/client/api/impl/AMRMClientImpl.java | 23 +-
.../hadoop/yarn/client/cli/ApplicationCLI.java | 7 +
.../hadoop/yarn/client/cli/RMAdminCLI.java | 191 +-
...stHedgingRequestRMFailoverProxyProvider.java | 31 +-
.../yarn/client/api/impl/TestAMRMClient.java | 22 +-
.../TestOpportunisticContainerAllocation.java | 71 +-
.../hadoop/yarn/client/cli/TestRMAdminCLI.java | 19 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 18 +-
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 23 +-
.../ApplicationClientProtocolPBClientImpl.java | 21 +-
.../ApplicationClientProtocolPBServiceImpl.java | 22 +
.../UpdateApplicationTimeoutsRequestPBImpl.java | 220 +
...UpdateApplicationTimeoutsResponsePBImpl.java | 73 +
.../ApplicationResourceUsageReportPBImpl.java | 28 +
.../RequestHedgingRMFailoverProxyProvider.java | 90 +-
.../client/api/impl/TimelineClientImpl.java | 8 +-
.../hadoop/yarn/event/AsyncDispatcher.java | 1 +
.../logaggregation/AggregatedLogFormat.java | 7 +-
.../yarn/logaggregation/LogCLIHelpers.java | 7 +
.../yarn/util/AbstractLivelinessMonitor.java | 17 +-
.../yarn/util/ResourceCalculatorPlugin.java | 39 +
.../java/org/apache/hadoop/yarn/util/Times.java | 33 +
.../org/apache/hadoop/yarn/webapp/WebApps.java | 8 +
.../static/dt-1.9.4/js/jquery.dataTables.min.js | 157 +
.../dt-1.9.4/js/jquery.dataTables.min.js.gz | Bin 21726 -> 0 bytes
.../webapps/static/jquery/jquery-1.8.2.min.js | 2 +
.../static/jquery/jquery-1.8.2.min.js.gz | Bin 33430 -> 0 bytes
.../static/jquery/jquery-ui-1.9.1.custom.min.js | 6 +
.../jquery/jquery-ui-1.9.1.custom.min.js.gz | Bin 62855 -> 0 bytes
.../webapps/static/jt/jquery.jstree.js | 4544 ++++++++++++++++++
.../webapps/static/jt/jquery.jstree.js.gz | Bin 37540 -> 0 bytes
.../src/main/resources/yarn-default.xml | 44 +-
.../hadoop/yarn/webapp/MyTestWebService.java | 3 +-
.../client/impl/zk/RegistrySecurity.java | 14 +-
.../registry/secure/KerberosConfiguration.java | 39 +-
.../registry/secure/TestSecureLogins.java | 99 +-
.../pom.xml | 4 +-
...pplicationHistoryManagerOnTimelineStore.java | 8 +-
.../ApplicationHistoryServer.java | 50 +-
.../webapp/AHSWebServices.java | 27 +-
.../timeline/webapp/TimelineWebServices.java | 23 +-
...pplicationHistoryManagerOnTimelineStore.java | 11 +-
.../webapp/TestAHSWebServices.java | 22 +-
.../TestRollingLevelDBTimelineStore.java | 6 +-
...TimelineAuthenticationFilterInitializer.java | 55 +-
.../webapp/TestTimelineWebServices.java | 141 +-
.../hadoop-yarn-server-common/pom.xml | 2 -
.../DistributedSchedulingAllocateResponse.java | 6 +-
.../protocolrecords/NodeHeartbeatResponse.java | 95 +-
...RegisterDistributedSchedulingAMResponse.java | 6 +-
.../RegisterNodeManagerRequest.java | 24 +
.../RegisterNodeManagerResponse.java | 40 +-
.../server/api/protocolrecords/RemoteNode.java | 90 +
...ributedSchedulingAllocateResponsePBImpl.java | 41 +-
.../impl/pb/NodeHeartbeatResponsePBImpl.java | 15 +-
...erDistributedSchedulingAMResponsePBImpl.java | 39 +-
.../pb/RegisterNodeManagerRequestPBImpl.java | 30 +-
.../pb/RegisterNodeManagerResponsePBImpl.java | 15 +-
.../impl/pb/RemoteNodePBImpl.java | 135 +
.../yarn/server/api/records/NodeStatus.java | 7 +-
.../records/OpportunisticContainersStatus.java | 152 +
.../api/records/QueuedContainersStatus.java | 45 -
.../api/records/impl/pb/NodeStatusPBImpl.java | 38 +-
.../pb/OpportunisticContainersStatusPBImpl.java | 139 +
.../impl/pb/QueuedContainersStatusPBImpl.java | 84 -
.../metrics/ApplicationMetricsConstants.java | 6 +
.../OpportunisticContainerAllocator.java | 79 +-
.../OpportunisticContainerContext.java | 14 +-
.../hadoop/yarn/server/utils/BuilderUtils.java | 5 +-
.../main/proto/yarn_server_common_protos.proto | 12 +-
.../yarn_server_common_service_protos.proto | 10 +-
.../hadoop/yarn/TestYarnServerApiClasses.java | 3 +
.../protocolrecords/TestProtocolRecords.java | 17 +-
.../hadoop-yarn-server-nodemanager/pom.xml | 6 +-
.../server/nodemanager/ContainerExecutor.java | 1 -
.../nodemanager/DefaultContainerExecutor.java | 24 +-
.../server/nodemanager/DeletionService.java | 13 +-
.../nodemanager/DockerContainerExecutor.java | 883 ----
.../nodemanager/LinuxContainerExecutor.java | 1 +
.../yarn/server/nodemanager/NodeManager.java | 4 +-
.../nodemanager/NodeResourceMonitorImpl.java | 6 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 34 +-
.../nodemanager/amrmproxy/AMRMProxyService.java | 4 +-
.../containermanager/ContainerManager.java | 3 +
.../containermanager/ContainerManagerImpl.java | 10 +-
.../application/ApplicationImpl.java | 6 +-
.../container/ContainerImpl.java | 29 +-
.../launcher/ContainerLaunch.java | 36 +-
.../runtime/DockerLinuxContainerRuntime.java | 14 +-
.../localizer/LocalCacheCleaner.java | 182 +
.../localizer/LocalResourcesTrackerImpl.java | 3 +-
.../localizer/LocalizedResource.java | 5 +-
.../localizer/ResourceLocalizationService.java | 93 +-
.../localizer/ResourceRetentionSet.java | 96 -
.../security/LocalizerTokenSelector.java | 4 +-
.../logaggregation/LogAggregationService.java | 15 +
.../monitor/ContainersMonitorImpl.java | 462 +-
.../queuing/QueuingContainerManagerImpl.java | 32 +
.../scheduler/DistributedScheduler.java | 22 +-
.../security/NMTokenSecretManagerInNM.java | 16 +-
.../nodemanager/util/ProcessIdFileReader.java | 12 +-
.../nodemanager/webapp/ContainerPage.java | 1 +
.../nodemanager/webapp/NMWebServices.java | 28 +-
.../nodemanager/webapp/dao/ContainerInfo.java | 7 +
.../container-executor/impl/configuration.c | 41 +-
.../container-executor/impl/configuration.h | 9 +
.../main/native/container-executor/impl/main.c | 5 +-
.../test/test-container-executor.c | 77 +-
.../TestDefaultContainerExecutor.java | 34 +
.../TestDockerContainerExecutor.java | 244 -
.../TestDockerContainerExecutorWithMocks.java | 305 --
.../nodemanager/TestNodeStatusUpdater.java | 6 +-
.../amrmproxy/MockResourceManagerFacade.java | 20 +-
.../BaseContainerManagerTest.java | 38 +-
.../localizer/TestLocalCacheCleanup.java | 235 +
.../TestLocalResourcesTrackerImpl.java | 4 +-
.../localizer/TestResourceRetention.java | 106 -
.../TestLogAggregationService.java | 28 +-
.../queuing/TestQueuingContainerManager.java | 6 +-
.../scheduler/TestDistributedScheduler.java | 20 +-
.../nodemanager/webapp/TestNMWebServices.java | 27 +-
.../webapp/TestNMWebServicesApps.java | 49 +-
.../webapp/TestNMWebServicesContainers.java | 27 +-
.../conf/capacity-scheduler.xml | 12 +
.../hadoop-yarn-server-resourcemanager/pom.xml | 16 +-
.../server/resourcemanager/AdminService.java | 6 +-
.../ApplicationMasterService.java | 3 +-
.../server/resourcemanager/ClientRMService.java | 137 +-
.../resourcemanager/NodesListManager.java | 4 +-
...pportunisticContainerAllocatorAMService.java | 58 +-
.../server/resourcemanager/RMAppManager.java | 39 +
.../server/resourcemanager/RMAuditLogger.java | 4 +-
.../yarn/server/resourcemanager/RMNMInfo.java | 2 +-
.../server/resourcemanager/RMServerUtils.java | 50 +-
.../server/resourcemanager/ResourceManager.java | 90 +-
.../resourcemanager/ResourceTrackerService.java | 3 +-
.../metrics/TimelineServiceV1Publisher.java | 4 +
.../metrics/TimelineServiceV2Publisher.java | 6 +
.../AbstractPreemptableResourceCalculator.java | 244 +
.../capacity/AbstractPreemptionEntity.java | 98 +
.../CapacitySchedulerPreemptionContext.java | 14 +
.../CapacitySchedulerPreemptionUtils.java | 119 +-
.../capacity/FifoCandidatesSelector.java | 127 +-
.../FifoIntraQueuePreemptionPlugin.java | 459 ++
.../capacity/IntraQueueCandidatesSelector.java | 238 +
.../IntraQueuePreemptionComputePlugin.java | 39 +
.../capacity/PreemptableResourceCalculator.java | 183 +-
.../capacity/PreemptionCandidatesSelector.java | 29 +-
.../ProportionalCapacityPreemptionPolicy.java | 86 +-
.../monitor/capacity/TempAppPerPartition.java | 101 +
.../monitor/capacity/TempQueuePerPartition.java | 142 +-
.../resourcemanager/recovery/RMStateStore.java | 38 +-
.../recovery/RMStateUpdateAppEvent.java | 15 +-
.../records/ApplicationAttemptStateData.java | 39 +-
.../recovery/records/ApplicationStateData.java | 27 +
.../pb/ApplicationAttemptStateDataPBImpl.java | 24 +
.../impl/pb/ApplicationStateDataPBImpl.java | 86 +
.../resourcemanager/resource/Priority.java | 31 -
.../server/resourcemanager/rmapp/RMApp.java | 3 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 77 +-
.../resourcemanager/rmapp/RMAppMetrics.java | 16 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 11 +-
.../rmapp/attempt/RMAppAttemptMetrics.java | 16 +
.../rmapp/monitor/RMAppLifetimeMonitor.java | 72 +-
.../rmcontainer/RMContainer.java | 13 +
.../rmcontainer/RMContainerImpl.java | 94 +-
.../server/resourcemanager/rmnode/RMNode.java | 10 +-
.../resourcemanager/rmnode/RMNodeImpl.java | 72 +-
.../rmnode/RMNodeStatusEvent.java | 6 +-
.../scheduler/AbstractYarnScheduler.java | 24 +-
.../scheduler/AppSchedulingInfo.java | 168 +-
.../scheduler/SchedulerApplicationAttempt.java | 75 +-
.../scheduler/activities/ActivitiesLogger.java | 17 +-
.../scheduler/activities/ActivitiesManager.java | 7 +-
.../scheduler/capacity/AbstractCSQueue.java | 71 +
...CSAMContainerLaunchDiagnosticsConstants.java | 2 +
.../scheduler/capacity/CSAssignment.java | 33 +
.../scheduler/capacity/CSQueue.java | 19 +-
.../scheduler/capacity/CapacityScheduler.java | 891 +++-
.../CapacitySchedulerConfiguration.java | 71 +
.../scheduler/capacity/LeafQueue.java | 540 ++-
.../scheduler/capacity/ParentQueue.java | 416 +-
.../allocator/AbstractContainerAllocator.java | 39 +-
.../capacity/allocator/ContainerAllocation.java | 12 +-
.../capacity/allocator/ContainerAllocator.java | 15 +-
.../allocator/IncreaseContainerAllocator.java | 89 +-
.../allocator/RegularContainerAllocator.java | 215 +-
.../scheduler/common/AssignmentInformation.java | 44 +-
.../common/ContainerAllocationProposal.java | 111 +
.../common/ResourceAllocationCommitter.java | 29 +
.../scheduler/common/ResourceCommitRequest.java | 164 +
.../scheduler/common/SchedulerContainer.java | 80 +
.../scheduler/common/fica/FiCaSchedulerApp.java | 640 ++-
.../distributed/NodeQueueLoadMonitor.java | 10 +-
.../scheduler/fair/FSAppAttempt.java | 132 +-
.../scheduler/fair/FSLeafQueue.java | 15 +-
.../scheduler/fair/FSParentQueue.java | 6 +-
.../scheduler/fair/FairScheduler.java | 4 +-
.../scheduler/fair/MaxRunningAppsEnforcer.java | 50 +-
.../fair/policies/FairSharePolicy.java | 31 +-
.../scheduler/fair/policies/FifoPolicy.java | 5 +-
.../scheduler/fifo/FifoAppAttempt.java | 110 +
.../scheduler/fifo/FifoScheduler.java | 55 +-
.../scheduler/placement/PlacementSet.java | 65 +
.../scheduler/placement/PlacementSetUtils.java | 36 +
.../placement/ResourceRequestUpdateResult.java | 43 +
.../placement/SchedulingPlacementSet.java | 90 +
.../scheduler/placement/SimplePlacementSet.java | 70 +
.../AbstractComparatorOrderingPolicy.java | 4 +-
.../scheduler/policy/FairOrderingPolicy.java | 3 +-
.../scheduler/policy/FifoOrderingPolicy.java | 4 +-
.../FifoOrderingPolicyForPendingApps.java | 3 +-
.../resourcemanager/webapp/NodesPage.java | 75 +-
.../resourcemanager/webapp/RMAppBlock.java | 8 +-
.../resourcemanager/webapp/RMWebServices.java | 121 +-
.../resourcemanager/webapp/dao/AppInfo.java | 34 +-
.../webapp/dao/ClusterMetricsInfo.java | 11 +-
.../resourcemanager/webapp/dao/NodeInfo.java | 41 +-
.../yarn_server_resourcemanager_recovery.proto | 3 +
.../yarn/server/resourcemanager/MockNM.java | 8 +-
.../yarn/server/resourcemanager/MockNodes.java | 21 +-
.../yarn/server/resourcemanager/MockRM.java | 91 +-
.../ReservationACLsTestBase.java | 7 +-
.../server/resourcemanager/TestAppManager.java | 13 +-
.../resourcemanager/TestClientRMService.java | 23 +-
...pportunisticContainerAllocatorAMService.java | 6 +-
.../yarn/server/resourcemanager/TestRM.java | 3 +-
.../resourcemanager/TestRMAdminService.java | 40 +-
.../yarn/server/resourcemanager/TestRMHA.java | 4 +-
.../resourcemanager/TestRMNodeTransitions.java | 40 +
.../resourcemanager/TestRMProxyUsersConf.java | 49 +-
.../server/resourcemanager/TestRMRestart.java | 19 +
.../resourcemanager/TestResourceManager.java | 6 +-
.../TestResourceTrackerService.java | 55 +
.../TestWorkPreservingRMRestart.java | 3 +-
.../ahs/TestRMApplicationHistoryWriter.java | 5 +-
.../applicationsmanager/MockAsm.java | 10 +-
.../metrics/TestSystemMetricsPublisher.java | 13 +-
.../TestSystemMetricsPublisherForV2.java | 4 +-
...alCapacityPreemptionPolicyMockFramework.java | 126 +-
...estProportionalCapacityPreemptionPolicy.java | 7 +-
...ionalCapacityPreemptionPolicyIntraQueue.java | 868 ++++
.../recovery/RMStateStoreTestBase.java | 4 +-
.../recovery/TestZKRMStateStore.java | 2 +-
.../planning/TestAlignedPlanner.java | 4 +-
.../planning/TestGreedyReservationAgent.java | 4 +-
.../server/resourcemanager/rmapp/MockRMApp.java | 6 +
.../rmapp/TestApplicationLifetimeMonitor.java | 150 +-
.../scheduler/TestSchedulerHealth.java | 22 +-
.../capacity/TestApplicationLimits.java | 185 +-
.../capacity/TestApplicationPriority.java | 101 +-
.../capacity/TestCapacityScheduler.java | 103 +-
.../TestCapacitySchedulerAsyncScheduling.java | 143 +
.../TestCapacitySchedulerNodeLabelUpdate.java | 74 +-
.../TestCapacitySchedulerQueueACLs.java | 3 +-
.../scheduler/capacity/TestChildQueueOrder.java | 21 +-
.../capacity/TestContainerAllocation.java | 45 +-
.../capacity/TestContainerResizing.java | 10 +-
.../scheduler/capacity/TestLeafQueue.java | 653 ++-
.../TestNodeLabelContainerAllocation.java | 4 +-
.../scheduler/capacity/TestParentQueue.java | 191 +-
.../scheduler/capacity/TestReservations.java | 283 +-
.../scheduler/capacity/TestUtils.java | 26 +
.../distributed/TestNodeQueueLoadMonitor.java | 8 +-
.../fair/TestContinuousScheduling.java | 189 +-
.../scheduler/fair/TestFairScheduler.java | 253 +-
.../fair/TestFairSchedulerEventLog.java | 2 +-
.../fair/TestFairSchedulerQueueACLs.java | 2 +-
.../fair/TestMaxRunningAppsEnforcer.java | 2 +-
.../scheduler/fair/TestSchedulingPolicy.java | 228 +
.../scheduler/fifo/TestFifoScheduler.java | 12 +-
.../security/TestAMRMTokens.java | 4 +
.../resourcemanager/webapp/TestAppPage.java | 3 +-
.../resourcemanager/webapp/TestNodesPage.java | 33 +
.../resourcemanager/webapp/TestRMWebApp.java | 1 +
.../webapp/TestRMWebAppFairScheduler.java | 3 +-
.../webapp/TestRMWebServiceAppsNodelabel.java | 230 +
.../webapp/TestRMWebServices.java | 46 +-
.../webapp/TestRMWebServicesApps.java | 153 +-
.../TestRMWebServicesAppsModification.java | 45 +-
.../webapp/TestRMWebServicesCapacitySched.java | 19 +-
...ebServicesDelegationTokenAuthentication.java | 6 +-
.../TestRMWebServicesDelegationTokens.java | 4 +-
.../webapp/TestRMWebServicesFairScheduler.java | 10 +-
.../TestRMWebServicesForCSWithPartitions.java | 13 +-
...tRMWebServicesHttpStaticUserPermissions.java | 4 +-
.../webapp/TestRMWebServicesNodeLabels.java | 55 +-
.../webapp/TestRMWebServicesNodes.java | 99 +-
.../webapp/TestRMWebServicesReservation.java | 18 +-
.../TestRMWebServicesSchedulerActivities.java | 99 +-
.../yarn/webapp/TestRMWithCSRFFilter.java | 7 +-
.../hadoop-yarn-server-tests/pom.xml | 6 +-
.../hadoop/yarn/server/TestDiskFailures.java | 4 +-
.../pom.xml | 2 +-
.../pom.xml | 24 +
...stTimelineReaderWebServicesHBaseStorage.java | 50 +-
.../storage/DataGeneratorForTest.java | 381 ++
.../storage/TestHBaseTimelineStorage.java | 3751 ---------------
.../storage/TestHBaseTimelineStorageApps.java | 1849 +++++++
.../TestHBaseTimelineStorageEntities.java | 1675 +++++++
.../hadoop-yarn-server-timelineservice/pom.xml | 22 +-
.../collector/AppLevelTimelineCollector.java | 14 +-
.../collector/TimelineCollectorWebService.java | 4 +-
.../reader/TimelineReaderServer.java | 25 +-
.../reader/TimelineReaderWebServices.java | 47 +-
.../reader/TestTimelineReaderWebServices.java | 73 +-
.../hadoop-yarn-server-web-proxy/pom.xml | 6 +-
.../server/webproxy/TestWebAppProxyServlet.java | 39 +-
.../site/markdown/DockerContainerExecutor.md.vm | 154 -
.../src/site/markdown/NodeLabel.md | 2 +-
.../src/site/markdown/NodeManagerRest.md | 2 +-
.../src/site/markdown/ResourceManagerRest.md | 609 ++-
.../src/site/markdown/TimelineServer.md | 21 +-
.../src/site/markdown/YarnCommands.md | 21 +-
.../src/site/markdown/YarnUI2.md | 55 +
.../hadoop-yarn/hadoop-yarn-ui/pom.xml | 193 +
.../hadoop-yarn-ui/public/crossdomain.xml | 15 +
.../hadoop-yarn-ui/src/main/webapp/.bowerrc | 4 +
.../hadoop-yarn-ui/src/main/webapp/.ember-cli | 9 +
.../hadoop-yarn-ui/src/main/webapp/.jshintrc | 32 +
.../src/main/webapp/.watchmanconfig | 3 +
.../src/main/webapp/WEB-INF/web.xml | 25 +
.../src/main/webapp/app/adapters/abstract.js | 48 +
.../main/webapp/app/adapters/cluster-info.js | 30 +
.../main/webapp/app/adapters/cluster-metric.js | 30 +
.../webapp/app/adapters/yarn-app-attempt.js | 38 +
.../src/main/webapp/app/adapters/yarn-app.js | 43 +
.../webapp/app/adapters/yarn-container-log.js | 80 +
.../main/webapp/app/adapters/yarn-container.js | 63 +
.../main/webapp/app/adapters/yarn-node-app.js | 53 +
.../webapp/app/adapters/yarn-node-container.js | 54 +
.../src/main/webapp/app/adapters/yarn-node.js | 33 +
.../src/main/webapp/app/adapters/yarn-queue.js | 30 +
.../main/webapp/app/adapters/yarn-rm-node.js | 36 +
.../hadoop-yarn-ui/src/main/webapp/app/app.js | 38 +
.../webapp/app/components/app-attempt-table.js | 22 +
.../src/main/webapp/app/components/app-table.js | 22 +
.../app/components/app-usage-donut-chart.js | 62 +
.../src/main/webapp/app/components/bar-chart.js | 127 +
.../app/components/base-chart-component.js | 144 +
.../app/components/base-usage-donut-chart.js | 43 +
.../webapp/app/components/breadcrumb-bar.js | 31 +
.../webapp/app/components/container-table.js | 22 +
.../main/webapp/app/components/donut-chart.js | 193 +
.../main/webapp/app/components/item-selector.js | 39 +
.../main/webapp/app/components/nodes-heatmap.js | 209 +
...er-app-memusage-by-nodes-stacked-barchart.js | 88 +
...app-ncontainers-by-nodes-stacked-barchart.js | 67 +
.../app/components/queue-configuration-table.js | 22 +
.../webapp/app/components/queue-navigator.js | 22 +
.../app/components/queue-usage-donut-chart.js | 69 +
.../main/webapp/app/components/queue-view.js | 289 ++
.../main/webapp/app/components/simple-table.js | 81 +
.../webapp/app/components/stacked-barchart.js | 198 +
.../main/webapp/app/components/timeline-view.js | 277 ++
.../main/webapp/app/components/tree-selector.js | 298 ++
.../src/main/webapp/app/config.js | 24 +
.../src/main/webapp/app/constants.js | 24 +
.../main/webapp/app/controllers/application.js | 55 +
.../webapp/app/controllers/cluster-overview.js | 32 +
.../webapp/app/controllers/yarn-app-attempt.js | 40 +
.../webapp/app/controllers/yarn-app-attempts.js | 40 +
.../src/main/webapp/app/controllers/yarn-app.js | 45 +
.../main/webapp/app/controllers/yarn-apps.js | 31 +
.../app/controllers/yarn-container-log.js | 40 +
.../webapp/app/controllers/yarn-node-app.js | 36 +
.../webapp/app/controllers/yarn-node-apps.js | 39 +
.../app/controllers/yarn-node-containers.js | 39 +
.../main/webapp/app/controllers/yarn-node.js | 37 +
.../app/controllers/yarn-nodes-heatmap.js | 36 +
.../main/webapp/app/controllers/yarn-nodes.js | 33 +
.../webapp/app/controllers/yarn-queue-apps.js | 46 +
.../main/webapp/app/controllers/yarn-queue.js | 44 +
.../main/webapp/app/controllers/yarn-queues.js | 34 +
.../webapp/app/controllers/yarn-services.js | 34 +
.../src/main/webapp/app/helpers/divide.js | 31 +
.../main/webapp/app/helpers/log-files-comma.js | 48 +
.../src/main/webapp/app/helpers/node-link.js | 37 +
.../src/main/webapp/app/helpers/node-menu.js | 66 +
.../src/main/webapp/app/helpers/node-name.js | 46 +
.../src/main/webapp/app/index.html | 44 +
.../src/main/webapp/app/initializers/env.js | 29 +
.../src/main/webapp/app/initializers/hosts.js | 28 +
.../src/main/webapp/app/initializers/loader.js | 86 +
.../src/main/webapp/app/models/cluster-info.js | 31 +
.../main/webapp/app/models/cluster-metric.js | 133 +
.../main/webapp/app/models/yarn-app-attempt.js | 143 +
.../src/main/webapp/app/models/yarn-app.js | 104 +
.../webapp/app/models/yarn-container-log.js | 25 +
.../main/webapp/app/models/yarn-container.js | 64 +
.../src/main/webapp/app/models/yarn-node-app.js | 44 +
.../webapp/app/models/yarn-node-container.js | 57 +
.../src/main/webapp/app/models/yarn-node.js | 33 +
.../src/main/webapp/app/models/yarn-queue.js | 94 +
.../src/main/webapp/app/models/yarn-rm-node.js | 99 +
.../src/main/webapp/app/models/yarn-user.js | 26 +
.../src/main/webapp/app/router.js | 58 +
.../src/main/webapp/app/routes/abstract.js | 32 +
.../src/main/webapp/app/routes/application.js | 40 +
.../main/webapp/app/routes/cluster-overview.js | 44 +
.../src/main/webapp/app/routes/index.js | 29 +
.../main/webapp/app/routes/yarn-app-attempt.js | 50 +
.../main/webapp/app/routes/yarn-app-attempts.js | 36 +
.../src/main/webapp/app/routes/yarn-app.js | 52 +
.../src/main/webapp/app/routes/yarn-apps.js | 35 +
.../main/webapp/app/routes/yarn-apps/apps.js | 22 +
.../webapp/app/routes/yarn-apps/services.js | 22 +
.../webapp/app/routes/yarn-container-log.js | 63 +
.../src/main/webapp/app/routes/yarn-node-app.js | 35 +
.../main/webapp/app/routes/yarn-node-apps.js | 35 +
.../webapp/app/routes/yarn-node-container.js | 36 +
.../webapp/app/routes/yarn-node-containers.js | 34 +
.../src/main/webapp/app/routes/yarn-node.js | 37 +
.../src/main/webapp/app/routes/yarn-nodes.js | 35 +
.../webapp/app/routes/yarn-nodes/heatmap.js | 22 +
.../main/webapp/app/routes/yarn-nodes/table.js | 22 +
.../main/webapp/app/routes/yarn-queue-apps.js | 42 +
.../src/main/webapp/app/routes/yarn-queue.js | 42 +
.../src/main/webapp/app/routes/yarn-queues.js | 42 +
.../main/webapp/app/routes/yarn-queues/index.js | 23 +
.../app/routes/yarn-queues/queues-selector.js | 25 +
.../main/webapp/app/serializers/cluster-info.js | 47 +
.../webapp/app/serializers/cluster-metric.js | 47 +
.../webapp/app/serializers/yarn-app-attempt.js | 75 +
.../src/main/webapp/app/serializers/yarn-app.js | 93 +
.../app/serializers/yarn-container-log.js | 39 +
.../webapp/app/serializers/yarn-container.js | 79 +
.../webapp/app/serializers/yarn-node-app.js | 83 +
.../app/serializers/yarn-node-container.js | 71 +
.../main/webapp/app/serializers/yarn-node.js | 56 +
.../main/webapp/app/serializers/yarn-queue.js | 145 +
.../main/webapp/app/serializers/yarn-rm-node.js | 74 +
.../src/main/webapp/app/services/env.js | 59 +
.../src/main/webapp/app/services/hosts.js | 74 +
.../src/main/webapp/app/styles/app.css | 279 ++
.../main/webapp/app/templates/application.hbs | 85 +
.../webapp/app/templates/cluster-overview.hbs | 150 +
.../templates/components/app-attempt-table.hbs | 62 +
.../app/templates/components/app-table.hbs | 86 +
.../app/templates/components/breadcrumb-bar.hbs | 22 +
.../templates/components/container-table.hbs | 54 +
.../templates/components/node-menu-panel.hbs | 44 +
.../app/templates/components/nodes-heatmap.hbs | 27 +
.../components/queue-configuration-table.hbs | 54 +
.../templates/components/queue-navigator.hbs | 28 +
.../app/templates/components/timeline-view.hbs | 54 +
.../src/main/webapp/app/templates/error.hbs | 19 +
.../src/main/webapp/app/templates/notfound.hbs | 20 +
.../webapp/app/templates/yarn-app-attempt.hbs | 43 +
.../webapp/app/templates/yarn-app-attempts.hbs | 55 +
.../src/main/webapp/app/templates/yarn-app.hbs | 253 +
.../src/main/webapp/app/templates/yarn-apps.hbs | 90 +
.../webapp/app/templates/yarn-apps/apps.hbs | 24 +
.../webapp/app/templates/yarn-apps/services.hbs | 27 +
.../webapp/app/templates/yarn-container-log.hbs | 38 +
.../main/webapp/app/templates/yarn-node-app.hbs | 62 +
.../webapp/app/templates/yarn-node-apps.hbs | 53 +
.../app/templates/yarn-node-container.hbs | 72 +
.../app/templates/yarn-node-containers.hbs | 60 +
.../src/main/webapp/app/templates/yarn-node.hbs | 118 +
.../main/webapp/app/templates/yarn-nodes.hbs | 70 +
.../webapp/app/templates/yarn-nodes/heatmap.hbs | 30 +
.../webapp/app/templates/yarn-nodes/table.hbs | 67 +
.../webapp/app/templates/yarn-queue-apps.hbs | 64 +
.../main/webapp/app/templates/yarn-queue.hbs | 108 +
.../main/webapp/app/templates/yarn-queues.hbs | 70 +
.../src/main/webapp/app/utils/color-utils.js | 67 +
.../src/main/webapp/app/utils/converter.js | 126 +
.../main/webapp/app/utils/href-address-utils.js | 29 +
.../src/main/webapp/app/utils/mock.js | 36 +
.../src/main/webapp/app/utils/sorter.js | 73 +
.../hadoop-yarn-ui/src/main/webapp/bower.json | 24 +
.../src/main/webapp/config/configs.env | 48 +
.../src/main/webapp/config/default-config.js | 32 +
.../src/main/webapp/config/environment.js | 70 +
.../src/main/webapp/ember-cli-build.js | 58 +
.../src/main/webapp/jsconfig.json | 6 +
.../hadoop-yarn-ui/src/main/webapp/package.json | 52 +
.../assets/images/datatables/Sorting icons.psd | Bin 0 -> 27490 bytes
.../public/assets/images/datatables/favicon.ico | Bin 0 -> 894 bytes
.../assets/images/datatables/sort_asc.png | Bin 0 -> 160 bytes
.../images/datatables/sort_asc_disabled.png | Bin 0 -> 148 bytes
.../assets/images/datatables/sort_both.png | Bin 0 -> 201 bytes
.../assets/images/datatables/sort_desc.png | Bin 0 -> 158 bytes
.../images/datatables/sort_desc_disabled.png | Bin 0 -> 146 bytes
.../webapp/public/assets/images/hadoop_logo.png | Bin 0 -> 31716 bytes
.../src/main/webapp/public/crossdomain.xml | 34 +
.../hadoop-yarn-ui/src/main/webapp/testem.json | 12 +
.../src/main/webapp/tests/helpers/resolver.js | 29 +
.../src/main/webapp/tests/helpers/start-app.js | 36 +
.../src/main/webapp/tests/index.html | 51 +
.../components/breadcrumb-bar-test.js | 43 +
.../src/main/webapp/tests/test-helper.js | 24 +
.../webapp/tests/unit/adapters/yarn-app-test.js | 30 +
.../unit/adapters/yarn-container-log-test.js | 73 +
.../tests/unit/adapters/yarn-node-app-test.js | 93 +
.../unit/adapters/yarn-node-container-test.js | 93 +
.../tests/unit/adapters/yarn-node-test.js | 42 +
.../tests/unit/adapters/yarn-rm-node-test.js | 44 +
.../unit/controllers/yarn-app-attempt-test.js | 30 +
.../unit/controllers/yarn-app-attempts-test.js | 30 +
.../tests/unit/controllers/yarn-app-test.js | 30 +
.../tests/unit/controllers/yarn-apps-test.js | 30 +
.../unit/controllers/yarn-container-log-test.js | 30 +
.../unit/controllers/yarn-node-app-test.js | 30 +
.../unit/controllers/yarn-node-apps-test.js | 30 +
.../controllers/yarn-node-containers-test.js | 30 +
.../tests/unit/controllers/yarn-node-test.js | 30 +
.../unit/controllers/yarn-nodes-heatmap-test.js | 30 +
.../tests/unit/controllers/yarn-nodes-test.js | 30 +
.../unit/controllers/yarn-queue-apps-test.js | 30 +
.../tests/unit/controllers/yarn-queues-test.js | 30 +
.../unit/controllers/yarn-services-test.js | 30 +
.../webapp/tests/unit/helpers/node-name-test.js | 28 +
.../webapp/tests/unit/initializers/env-test.js | 41 +
.../tests/unit/initializers/hosts-test.js | 41 +
.../tests/unit/initializers/jquery-test.js | 41 +
.../tests/unit/initializers/loader-test.js | 40 +
.../webapp/tests/unit/mixins/charts-test.js | 30 +
.../webapp/tests/unit/models/yarn-app-test.js | 30 +
.../unit/models/yarn-container-log-test.js | 48 +
.../tests/unit/models/yarn-node-app-test.js | 65 +
.../unit/models/yarn-node-container-test.js | 78 +
.../webapp/tests/unit/models/yarn-node-test.js | 58 +
.../tests/unit/models/yarn-rm-node-test.js | 95 +
.../tests/unit/routes/yarn-app-attempts-test.js | 29 +
.../webapp/tests/unit/routes/yarn-apps-test.js | 29 +
.../unit/routes/yarn-container-log-test.js | 120 +
.../tests/unit/routes/yarn-node-app-test.js | 56 +
.../tests/unit/routes/yarn-node-apps-test.js | 60 +
.../unit/routes/yarn-node-container-test.js | 61 +
.../unit/routes/yarn-node-containers-test.js | 68 +
.../webapp/tests/unit/routes/yarn-node-test.js | 84 +
.../webapp/tests/unit/routes/yarn-nodes-test.js | 74 +
.../tests/unit/routes/yarn-queue-apps-test.js | 29 +
.../tests/unit/routes/yarn-queues-test.js | 29 +
.../tests/unit/serializers/yarn-app-test.js | 33 +
.../unit/serializers/yarn-container-log-test.js | 49 +
.../unit/serializers/yarn-node-app-test.js | 102 +
.../serializers/yarn-node-container-test.js | 128 +
.../tests/unit/serializers/yarn-node-test.js | 69 +
.../tests/unit/serializers/yarn-rm-node-test.js | 153 +
.../main/webapp/tests/unit/services/env-test.js | 30 +
.../webapp/tests/unit/services/hosts-test.js | 30 +
.../webapp/tests/unit/utils/converter-test.js | 52 +
.../main/webapp/tests/unit/utils/sorter-test.js | 26 +
hadoop-yarn-project/hadoop-yarn/pom.xml | 1 +
pom.xml | 8 +-
1005 files changed, 49211 insertions(+), 14325 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e55bdefd/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
index 47692e2,8ce5f3f..9c8dd1b
--- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
@@@ -111,10 -111,10 +111,14 @@@ http://maven.apache.org/xsd/maven-4.0.0
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
</dependencies>
<build>
@@@ -162,10 -161,8 +165,9 @@@
<include>inotify.proto</include>
<include>erasurecoding.proto</include>
<include>ReconfigurationProtocol.proto</include>
+ <include>DatanodeContainerProtocol.proto</include>
</includes>
</source>
-
<output>${project.build.directory}/generated-sources/java</output>
</configuration>
</execution>
</executions>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e55bdefd/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 3693a73,5427db8..2bbf253
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@@ -352,12 -349,20 +350,23 @@@ http://maven.apache.org/xsd/maven-4.0.0
<include>QJournalProtocol.proto</include>
<include>editlog.proto</include>
<include>fsimage.proto</include>
+ <include>StorageContainerLocationProtocol.proto</include>
+ <include>StorageContainerDatanodeProtocol.proto</include>
+ <include>CBlockServiceProtocol.proto</include>
</includes>
</source>
-
<output>${project.build.directory}/generated-sources/java</output>
+ </configuration>
+ </execution>
+ <execution>
+ <id>resource-gz</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>resource-gz</goal>
+ </goals>
+ <configuration>
+
<inputDirectory>${basedir}/src/main/webapps/static</inputDirectory>
+
<outputDirectory>${basedir}/target/webapps/static</outputDirectory>
+ <extensions>js,css</extensions>
</configuration>
</execution>
</executions>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e55bdefd/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e55bdefd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e55bdefd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
index e4c1989,0000000..c251966
mode 100644,000000..100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
@@@ -1,92 -1,0 +1,94 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.impl;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.common.interfaces
+ .ContainerLocationManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Path;
++import java.nio.file.Paths;
+import java.util.List;
+
+/**
+ * A class that tells the ContainerManager where to place the containers.
+ * Please note : There is *no* one-to-one correlation between metadata
+ * metadataLocations and data metadataLocations.
+ *
+ * For example : A user could map all container files to a
+ * SSD but leave data/metadata on bunch of other disks.
+ */
+public class ContainerLocationManagerImpl implements ContainerLocationManager
{
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerLocationManagerImpl.class);
+
+ private final List<StorageLocation> dataLocations;
+ private int currentIndex;
+ private final List<StorageLocation> metadataLocations;
+
+ /**
+ * Constructs a Location Manager.
+ * @param metadataLocations - Refers to the metadataLocations
+ * where we store the container metadata.
+ * @param dataDirs - metadataLocations where we store the actual
+ * data or chunk files.
+ * @throws IOException
+ */
+ public ContainerLocationManagerImpl(List<StorageLocation> metadataLocations,
+ List<StorageLocation> dataDirs)
+ throws IOException {
+ dataLocations = dataDirs;
+ this.metadataLocations = metadataLocations;
+ }
+ /**
+ * Returns the path where the container should be placed from a set of
+ * metadataLocations.
+ *
+ * @return A path where we should place this container and metadata.
+ * @throws IOException
+ */
+ @Override
+ public Path getContainerPath()
+ throws IOException {
+ Preconditions.checkState(metadataLocations.size() > 0);
+ int index = currentIndex % metadataLocations.size();
- Path path = metadataLocations.get(index).getFile().toPath();
++ Path path = Paths.get(metadataLocations.get(index).getNormalizedUri());
+ return path.resolve(OzoneConsts.CONTAINER_ROOT_PREFIX);
+ }
+
+ /**
+ * Returns the path where the container Data file are stored.
+ *
+ * @return a path where we place the LevelDB and data files of a container.
+ * @throws IOException
+ */
+ @Override
+ public Path getDataPath(String containerName) throws IOException {
- Path currentPath = dataLocations.get(currentIndex++ %
dataLocations.size())
- .getFile().toPath();
++ Path currentPath = Paths.get(
++ dataLocations.get(currentIndex++ % dataLocations.size())
++ .getNormalizedUri());
+ currentPath = currentPath.resolve(OzoneConsts.CONTAINER_PREFIX);
+ return currentPath.resolve(containerName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e55bdefd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
index 88bdebc,0000000..cb571ea
mode 100644,000000..100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
@@@ -1,565 -1,0 +1,565 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.impl;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
+import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
+import org.apache.hadoop.ozone.container.common.interfaces.ChunkManager;
+import org.apache.hadoop.ozone.container.common.interfaces
+ .ContainerLocationManager;
+import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
+import org.apache.hadoop.ozone.container.common.interfaces.KeyManager;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.DigestInputStream;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.ConcurrentNavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_EXTENSION;
+import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_META;
+
+/**
+ * A Generic ContainerManagerImpl that will be called from Ozone
+ * ContainerManagerImpl. This allows us to support delta changes to ozone
+ * version without having to rewrite the containerManager.
+ */
+public class ContainerManagerImpl implements ContainerManager {
+ static final Logger LOG =
+ LoggerFactory.getLogger(ContainerManagerImpl.class);
+
+ private final ConcurrentSkipListMap<String, ContainerStatus>
+ containerMap = new ConcurrentSkipListMap<>();
+
+ // This lock follows fair locking policy of first-come first-serve
+ // for waiting threads.
+ private final ReentrantReadWriteLock lock = new
ReentrantReadWriteLock(true);
+ private ContainerLocationManager locationManager;
+ private ChunkManager chunkManager;
+ private KeyManager keyManager;
+
+ /**
+ * Init call that sets up a container Manager.
+ *
+ * @param config - Configuration.
+ * @param containerDirs - List of Metadata Container locations.
+ * @throws IOException
+ */
+ @Override
+ public void init(
+ Configuration config, List<StorageLocation> containerDirs)
+ throws IOException {
+ Preconditions.checkNotNull(config);
+ Preconditions.checkNotNull(containerDirs);
+ Preconditions.checkState(containerDirs.size() > 0);
+
+ readLock();
+ try {
+ for (StorageLocation path : containerDirs) {
- File directory = path.getFile();
++ File directory = Paths.get(path.getNormalizedUri()).toFile();
+ if (!directory.isDirectory()) {
+ LOG.error("Invalid path to container metadata directory. path: {}",
+ path.toString());
+ throw new IOException("Invalid path to container metadata
directory" +
+ ". " + path);
+ }
+ File[] files = directory.listFiles(new ContainerFilter());
+ if (files != null) {
+ for (File containerFile : files) {
+ String containerPath =
+ ContainerUtils.getContainerNameFromFile(containerFile);
+ Preconditions.checkNotNull(containerPath);
+ readContainerInfo(containerPath);
+ }
+ }
+ }
+
+ List<StorageLocation> dataDirs = new LinkedList<>();
+ for (String dir : config.getStrings(DFS_DATANODE_DATA_DIR_KEY)) {
+ StorageLocation location = StorageLocation.parse(dir);
+ dataDirs.add(location);
+ }
+ this.locationManager =
+ new ContainerLocationManagerImpl(containerDirs, dataDirs);
+
+ } finally {
+ readUnlock();
+ }
+ }
+
+ /**
+ * Reads the Container Info from a file and verifies that checksum match. If
+ * the checksums match, then that file is added to containerMap.
+ *
+ * @param containerName - Name which points to the persisted container.
+ */
+ private void readContainerInfo(String containerName)
+ throws IOException {
+ Preconditions.checkState(containerName.length() > 0);
+ FileInputStream containerStream = null;
+ DigestInputStream dis = null;
+ FileInputStream metaStream = null;
+ Path cPath = Paths.get(containerName).getFileName();
+ String keyName = null;
+ if (cPath != null) {
+ keyName = cPath.toString();
+ }
+ Preconditions.checkNotNull(keyName);
+
+ try {
+ String containerFileName = containerName.concat(CONTAINER_EXTENSION);
+ String metaFileName = containerName.concat(CONTAINER_META);
+
+ containerStream = new FileInputStream(containerFileName);
+
+ metaStream = new FileInputStream(metaFileName);
+
+ MessageDigest sha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
+
+ dis = new DigestInputStream(containerStream, sha);
+
+ ContainerData containerData = ContainerData.getFromProtBuf(
+ ContainerProtos.ContainerData.parseDelimitedFrom(dis));
+
+
+ ContainerProtos.ContainerMeta meta = ContainerProtos.ContainerMeta
+ .parseDelimitedFrom(metaStream);
+
+ if (meta != null && !DigestUtils.sha256Hex(sha.digest()).equals(meta
+ .getHash())) {
+ throw new IOException("Invalid SHA found for file.");
+ }
+
+ containerMap.put(keyName, new ContainerStatus(containerData, true));
+
+ } catch (IOException | NoSuchAlgorithmException ex) {
+ LOG.error("read failed for file: {} ex: {}",
+ containerName, ex.getMessage());
+
+ // TODO : Add this file to a recovery Queue.
+
+ // Remember that this container is busted and we cannot use it.
+ containerMap.put(keyName, new ContainerStatus(null, false));
+ } finally {
+ IOUtils.closeStream(dis);
+ IOUtils.closeStream(containerStream);
+ IOUtils.closeStream(metaStream);
+ }
+ }
+
+ /**
+ * Creates a container with the given name.
+ *
+ * @param pipeline -- Nodes which make up this container.
+ * @param containerData - Container Name and metadata.
+ * @throws IOException
+ */
+ @Override
+ public void createContainer(Pipeline pipeline, ContainerData containerData)
+ throws IOException {
+ Preconditions.checkNotNull(containerData);
+
+ writeLock();
+ try {
+ if (containerMap.containsKey(containerData.getName())) {
+ throw new FileAlreadyExistsException("container already exists.");
+ }
+
+ // This is by design. We first write and close the
+ // container Info and metadata to a directory.
+ // Then read back and put that info into the containerMap.
+ // This allows us to make sure that our write is consistent.
+
+ writeContainerInfo(containerData);
+ File cFile = new File(containerData.getContainerPath());
+ readContainerInfo(ContainerUtils.getContainerNameFromFile(cFile));
+ } catch (NoSuchAlgorithmException ex) {
+ throw new IOException("failed to create container", ex);
+
+ } finally {
+ writeUnlock();
+ }
+
+ }
+
+ /**
+ * Writes a container to a chosen location and updates the container Map.
+ *
+ * The file formats of ContainerData and Container Meta is the following.
+ *
+ * message ContainerData {
+ * required string name = 1;
+ * repeated KeyValue metadata = 2;
+ * optional string dbPath = 3;
+ * optional string containerPath = 4;
+ * }
+ *
+ * message ContainerMeta {
+ * required string fileName = 1;
+ * required string hash = 2;
+ * }
+ *
+ * @param containerData - container Data
+ */
+ private void writeContainerInfo(ContainerData containerData)
+ throws IOException, NoSuchAlgorithmException {
+
+ Preconditions.checkNotNull(this.locationManager);
+
+ FileOutputStream containerStream = null;
+ DigestOutputStream dos = null;
+ FileOutputStream metaStream = null;
+ Path location = locationManager.getContainerPath();
+
+ File containerFile = ContainerUtils.getContainerFile(containerData,
+ location);
+ File metadataFile = ContainerUtils.getMetadataFile(containerData,
location);
+
+ try {
+ ContainerUtils.verifyIsNewContainer(containerFile, metadataFile);
+
+ Path metadataPath = this.locationManager.getDataPath(
+ containerData.getContainerName());
+ metadataPath = ContainerUtils.createMetadata(metadataPath);
+
+ containerStream = new FileOutputStream(containerFile);
+ metaStream = new FileOutputStream(metadataFile);
+ MessageDigest sha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
+
+ dos = new DigestOutputStream(containerStream, sha);
+ containerData.setDBPath(metadataPath.resolve(OzoneConsts.CONTAINER_DB)
+ .toString());
+ containerData.setContainerPath(containerFile.toString());
+
+ ContainerProtos.ContainerData protoData = containerData
+ .getProtoBufMessage();
+ protoData.writeDelimitedTo(dos);
+
+ ContainerProtos.ContainerMeta protoMeta = ContainerProtos
+ .ContainerMeta.newBuilder()
+ .setFileName(containerFile.toString())
+ .setHash(DigestUtils.sha256Hex(sha.digest()))
+ .build();
+ protoMeta.writeDelimitedTo(metaStream);
+
+ } catch (IOException ex) {
+
+ // TODO : we need to clean up partially constructed files
+ // The proper way to do would be for a thread
+ // to read all these 3 artifacts and make sure they are
+ // sane. That info needs to come from the replication
+ // pipeline, and if not consistent delete these file.
+
+ // In case of ozone this is *not* a deal breaker since
+ // SCM is guaranteed to generate unique container names.
+
+ LOG.error("creation of container failed. Name: {} ",
+ containerData.getContainerName());
+ throw ex;
+ } finally {
+ IOUtils.closeStream(dos);
+ IOUtils.closeStream(containerStream);
+ IOUtils.closeStream(metaStream);
+ }
+ }
+
+
+
+ /**
+ * Deletes an existing container.
+ *
+ * @param pipeline - nodes that make this container.
+ * @param containerName - name of the container.
+ * @throws IOException
+ */
+ @Override
+ public void deleteContainer(Pipeline pipeline, String containerName) throws
+ IOException {
+ Preconditions.checkState(containerName.length() > 0);
+ writeLock();
+ try {
+ ContainerStatus status = containerMap.get(containerName);
+ if (status == null) {
+ LOG.info("No such container. Name: {}", containerName);
+ throw new IOException("No such container. Name : " + containerName);
+ }
+ ContainerUtils.removeContainer(status.containerData);
+ containerMap.remove(containerName);
+ } finally {
+ writeUnlock();
+ }
+
+ }
+
+ /**
+ * A simple interface for container Iterations.
+ * <p/>
+ * This call make no guarantees about consistency of the data between
+ * different list calls. It just returns the best known data at that point
of
+ * time. It is possible that using this iteration you can miss certain
+ * container from the listing.
+ *
+ * @param prefix - Return keys that match this prefix.
+ * @param count - how many to return
+ * @param prevKey - Previous Key Value or empty String.
+ * @param data - Actual containerData
+ * @throws IOException
+ */
+ @Override
+ public void listContainer(String prefix, long count, String prevKey,
+ List<ContainerData> data) throws IOException {
+ // TODO : Support list with Prefix and PrevKey
+ Preconditions.checkNotNull(data);
+ readLock();
+ try {
+ ConcurrentNavigableMap<String, ContainerStatus> map = null;
+ if (prevKey == null || prevKey.isEmpty()) {
+ map = containerMap.tailMap(containerMap.firstKey(), true);
+ } else {
+ map = containerMap.tailMap(prevKey, false);
+ }
+
+ int currentCount = 0;
+ for (ContainerStatus entry : map.values()) {
+ if (currentCount < count) {
+ data.add(entry.getContainer());
+ currentCount++;
+ } else {
+ return;
+ }
+ }
+ } finally {
+ readUnlock();
+ }
+ }
+
+ /**
+ * Get metadata about a specific container.
+ *
+ * @param containerName - Name of the container
+ * @return ContainerData - Container Data.
+ * @throws IOException
+ */
+ @Override
+ public ContainerData readContainer(String containerName) throws IOException
{
+ if(!containerMap.containsKey(containerName)) {
+ throw new IOException("Unable to find the container. Name: "
+ + containerName);
+ }
+ return containerMap.get(containerName).getContainer();
+ }
+
+ /**
+ * Supports clean shutdown of container.
+ *
+ * @throws IOException
+ */
+ @Override
+ public void shutdown() {
+ Preconditions.checkState(this.hasWriteLock());
+ this.containerMap.clear();
+ }
+
+
+ @VisibleForTesting
+ ConcurrentSkipListMap<String, ContainerStatus> getContainerMap() {
+ return containerMap;
+ }
+
+ /**
+ * Acquire read lock.
+ */
+ @Override
+ public void readLock() {
+ this.lock.readLock().lock();
+
+ }
+
+ /**
+ * Release read lock.
+ */
+ @Override
+ public void readUnlock() {
+ this.lock.readLock().unlock();
+ }
+
+ /**
+ * Check if the current thread holds read lock.
+ */
+ @Override
+ public boolean hasReadLock() {
+ return this.lock.readLock().tryLock();
+ }
+
+ /**
+ * Acquire write lock.
+ */
+ @Override
+ public void writeLock() {
+ this.lock.writeLock().lock();
+ }
+
+ /**
+ * Acquire write lock, unless interrupted while waiting.
+ */
+ @Override
+ public void writeLockInterruptibly() throws InterruptedException {
+ this.lock.writeLock().lockInterruptibly();
+
+ }
+
+ /**
+ * Release write lock.
+ */
+ @Override
+ public void writeUnlock() {
+ this.lock.writeLock().unlock();
+
+ }
+
+ /**
+ * Check if the current thread holds write lock.
+ */
+ @Override
+ public boolean hasWriteLock() {
+ return this.lock.writeLock().isHeldByCurrentThread();
+ }
+
+ /**
+ * Sets the chunk Manager.
+ * @param chunkManager
+ */
+ public void setChunkManager(ChunkManager chunkManager) {
+ this.chunkManager = chunkManager;
+ }
+
+ public ChunkManager getChunkManager() {
+ return this.chunkManager;
+ }
+
+ /**
+ * Sets the Key Manager.
+ *
+ * @param keyManager - Key Manager.
+ */
+ @Override
+ public void setKeyManager(KeyManager keyManager) {
+ this.keyManager = keyManager;
+ }
+
+ /**
+ * Gets the Key Manager.
+ *
+ * @return KeyManager.
+ */
+ @Override
+ public KeyManager getKeyManager() {
+ return this.keyManager;
+ }
+
+ /**
+ * Filter out only container files from the container metadata dir.
+ */
+ private static class ContainerFilter implements FilenameFilter {
+ /**
+ * Tests if a specified file should be included in a file list.
+ *
+ * @param dir the directory in which the file was found.
+ * @param name the name of the file.
+ * @return <code>true</code> if and only if the name should be included in
+ * the file list; <code>false</code> otherwise.
+ */
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.endsWith(CONTAINER_EXTENSION);
+ }
+ }
+
+ /**
+ * This is an immutable class that represents the state of a container. if
the
+ * container reading encountered an error when we boot up we will post that
+ * info to a recovery queue and keep the info in the containerMap.
+ * <p/>
+ * if and when the issue is fixed, the expectation is that this entry will
be
+ * deleted by the recovery thread from the containerMap and will insert
entry
+ * instead of modifying this class.
+ */
+ @VisibleForTesting
+ static class ContainerStatus {
+ private final ContainerData containerData;
+ private final boolean active;
+
+ /**
+ * Creates a Container Status class.
+ *
+ * @param containerData - ContainerData.
+ * @param active - Active or not active.
+ */
+ ContainerStatus(ContainerData containerData, boolean active) {
+ this.containerData = containerData;
+ this.active = active;
+ }
+
+ /**
+ * Returns container if it is active. It is not active if we have had an
+ * error and we are waiting for the background threads to fix the issue.
+ *
+ * @return ContainerData.
+ */
+ public ContainerData getContainer() {
+ if (active) {
+ return containerData;
+ }
+ return null;
+ }
+
+ /**
+ * Indicates if a container is Active.
+ *
+ * @return
+ */
+ public boolean isActive() {
+ return active;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
