Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64ed2aa4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64ed2aa4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64ed2aa4 Branch: refs/heads/HDFS-12943 Commit: 64ed2aa4325214f6f5f1889da935dc68f0d51122 Parents: f7f2739 6b21a59 Author: Erik Krogen <[email protected]> Authored: Fri Jun 1 09:43:42 2018 -0700 Committer: Erik Krogen <[email protected]> Committed: Fri Jun 1 09:43:42 2018 -0700 ---------------------------------------------------------------------- .gitignore | 8 + dev-support/bin/dist-layout-stitching | 15 - dev-support/bin/ozone-dist-layout-stitching | 157 ++++ dev-support/bin/ozone-dist-tar-stitching | 48 ++ .../resources/assemblies/hadoop-yarn-dist.xml | 2 +- .../org/apache/hadoop/conf/Configuration.java | 20 +- .../apache/hadoop/crypto/key/KeyProvider.java | 18 + .../crypto/key/kms/KMSClientProvider.java | 229 +++--- .../crypto/key/kms/KMSDelegationToken.java | 22 +- .../crypto/key/kms/KMSLegacyTokenRenewer.java | 56 -- .../hadoop/crypto/key/kms/KMSTokenRenewer.java | 103 --- .../hadoop/crypto/key/kms/package-info.java | 18 - .../fs/CommonConfigurationKeysPublic.java | 32 +- .../java/org/apache/hadoop/fs/FileUtil.java | 18 +- .../apache/hadoop/ha/ZKFailoverController.java | 2 +- .../org/apache/hadoop/http/HttpServer2.java | 79 +- .../apache/hadoop/io/ElasticByteBufferPool.java | 12 + .../main/java/org/apache/hadoop/io/IOUtils.java | 27 + .../org/apache/hadoop/io/ReadaheadPool.java | 6 +- .../org/apache/hadoop/net/NetworkTopology.java | 2 +- .../hadoop/security/UserGroupInformation.java | 8 +- .../web/DelegationTokenAuthenticatedURL.java | 21 +- .../DelegationTokenAuthenticationHandler.java | 8 +- .../web/DelegationTokenAuthenticator.java | 2 +- .../org/apache/hadoop/util/DiskChecker.java | 46 +- .../java/org/apache/hadoop/util/KMSUtil.java | 45 +- .../hadoop/util/KMSUtilFaultInjector.java | 49 -- .../java/org/apache/hadoop/util/RunJar.java | 27 +- .../main/java/org/apache/hadoop/util/Shell.java | 15 + .../org/apache/hadoop/util/StringUtils.java | 2 +- .../main/java/org/apache/hadoop/util/Time.java | 12 + ...apache.hadoop.security.token.TokenIdentifier | 1 - ...rg.apache.hadoop.security.token.TokenRenewer | 3 +- .../src/main/resources/core-default.xml | 65 +- .../markdown/release/2.9.1/CHANGES.2.9.1.md | 277 +++++++ .../release/2.9.1/RELEASENOTES.2.9.1.md | 88 ++ .../apache/hadoop/conf/TestConfiguration.java | 4 +- .../apache/hadoop/crypto/TestCryptoStreams.java | 39 +- .../crypto/key/kms/TestKMSClientProvider.java | 162 ---- .../kms/TestLoadBalancingKMSClientProvider.java | 67 +- .../java/org/apache/hadoop/fs/TestFileUtil.java | 40 +- .../fs/TestRawLocalFileSystemContract.java | 2 +- .../java/org/apache/hadoop/fs/TestTrash.java | 10 +- .../hadoop/fs/contract/ContractTestUtils.java | 29 + .../org/apache/hadoop/http/TestHttpServer.java | 61 ++ .../TestRollingFileSystemSinkWithLocal.java | 19 +- .../org/apache/hadoop/util/TestDiskChecker.java | 102 --- .../hadoop/util/TestDiskCheckerWithDiskIo.java | 173 ++++ .../org/apache/hadoop/util/TestKMSUtil.java | 65 -- .../java/org/apache/hadoop/util/TestRunJar.java | 79 +- .../org/apache/hadoop/util/TestStringUtils.java | 2 +- .../hadoop/crypto/key/kms/server/KMSACLs.java | 4 +- .../src/main/resources/kms-default.xml | 35 + .../hadoop/crypto/key/kms/server/TestKMS.java | 519 ++---------- hadoop-dist/pom.xml | 164 ++-- hadoop-dist/src/main/compose/ozone/.env | 2 +- .../src/main/compose/ozone/docker-compose.yaml | 8 +- hadoop-hdds/client/pom.xml | 5 +- .../apache/hadoop/hdds/scm/XceiverClient.java | 40 +- .../hadoop/hdds/scm/XceiverClientGrpc.java | 217 +++++ .../hadoop/hdds/scm/XceiverClientHandler.java | 10 +- .../hdds/scm/XceiverClientInitializer.java | 18 +- .../hadoop/hdds/scm/XceiverClientManager.java | 38 +- .../hadoop/hdds/scm/XceiverClientMetrics.java | 10 +- .../hadoop/hdds/scm/XceiverClientRatis.java | 18 +- .../scm/client/ContainerOperationClient.java | 126 +-- .../hadoop/hdds/scm/client/HddsClientUtils.java | 23 + .../hdds/scm/storage/ChunkInputStream.java | 19 +- .../hdds/scm/storage/ChunkOutputStream.java | 26 +- .../common/dev-support/findbugsExcludeFile.xml | 3 + hadoop-hdds/common/pom.xml | 71 +- .../org/apache/hadoop/hdds/client/BlockID.java | 71 ++ .../hadoop/hdds/conf/HddsConfServlet.java | 2 +- .../hadoop/hdds/conf/OzoneConfiguration.java | 6 +- .../hadoop/hdds/protocol/DatanodeDetails.java | 219 +++-- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 17 + .../hadoop/hdds/scm/XceiverClientSpi.java | 4 +- .../hadoop/hdds/scm/client/ScmClient.java | 49 +- .../hadoop/hdds/scm/container/ContainerID.java | 2 +- .../common/helpers/AllocatedBlock.java | 20 +- .../container/common/helpers/ContainerInfo.java | 48 +- .../common/helpers/DeleteBlockResult.java | 16 +- .../scm/container/common/helpers/Pipeline.java | 18 +- .../common/helpers/PipelineChannel.java | 2 + .../helpers/StorageContainerException.java | 2 +- .../scm/protocol/ScmBlockLocationProtocol.java | 12 - .../StorageContainerLocationProtocol.java | 26 +- ...kLocationProtocolClientSideTranslatorPB.java | 46 +- ...rLocationProtocolClientSideTranslatorPB.java | 60 +- .../scm/storage/ContainerProtocolCalls.java | 110 ++- .../apache/hadoop/ozone/OzoneConfigKeys.java | 21 + .../org/apache/hadoop/ozone/OzoneConsts.java | 12 +- .../apache/hadoop/ozone/common/BlockGroup.java | 26 +- .../ozone/common/DeleteBlockGroupResult.java | 13 +- .../container/common/helpers/ChunkInfo.java | 7 +- .../ozone/container/common/helpers/KeyData.java | 48 +- ...kLocationProtocolServerSideTranslatorPB.java | 38 +- ...rLocationProtocolServerSideTranslatorPB.java | 30 +- .../org/apache/hadoop/utils/LevelDBStore.java | 4 +- .../apache/hadoop/utils/MetadataKeyFilters.java | 20 +- .../org/apache/hadoop/utils/RocksDBStore.java | 1 + .../main/java/org/apache/ratis/RatisHelper.java | 6 +- .../com/google/protobuf/ShadedProtoUtil.java | 38 - .../com/google/protobuf/package-info.java | 22 - .../main/proto/DatanodeContainerProtocol.proto | 143 ++-- .../main/proto/ScmBlockLocationProtocol.proto | 36 +- .../StorageContainerLocationProtocol.proto | 16 +- hadoop-hdds/common/src/main/proto/hdds.proto | 23 +- .../common/src/main/resources/ozone-default.xml | 44 +- hadoop-hdds/container-service/pom.xml | 4 +- .../container/common/helpers/ChunkUtils.java | 56 +- .../container/common/helpers/ContainerData.java | 278 +++++-- .../common/helpers/ContainerMetrics.java | 16 +- .../common/helpers/ContainerReport.java | 19 +- .../common/helpers/ContainerUtils.java | 51 +- .../helpers/DeletedContainerBlocksSummary.java | 19 +- .../container/common/helpers/FileUtils.java | 7 +- .../container/common/helpers/KeyUtils.java | 16 +- .../container/common/impl/ChunkManagerImpl.java | 86 +- .../impl/ContainerLocationManagerImpl.java | 18 +- .../common/impl/ContainerManagerImpl.java | 463 ++++++----- .../common/impl/ContainerReportManagerImpl.java | 43 +- .../container/common/impl/ContainerStatus.java | 217 ----- .../common/impl/ContainerStorageLocation.java | 9 + .../ozone/container/common/impl/Dispatcher.java | 152 ++-- .../container/common/impl/KeyManagerImpl.java | 72 +- .../RandomContainerDeletionChoosingPolicy.java | 10 +- .../common/impl/StorageLocationReport.java | 245 +++++- ...NOrderedContainerDeletionChoosingPolicy.java | 20 +- .../common/interfaces/ChunkManager.java | 21 +- .../ContainerDeletionChoosingPolicy.java | 3 +- .../common/interfaces/ContainerDispatcher.java | 4 +- .../ContainerLocationManagerMXBean.java | 4 +- .../common/interfaces/ContainerManager.java | 93 +-- .../interfaces/ContainerReportManager.java | 8 +- .../container/common/interfaces/KeyManager.java | 17 +- .../interfaces/StorageLocationReportMXBean.java | 40 + .../statemachine/DatanodeStateMachine.java | 11 +- .../common/statemachine/StateContext.java | 54 +- .../background/BlockDeletingService.java | 13 +- .../CloseContainerCommandHandler.java | 113 +++ .../commandhandler/CloseContainerHandler.java | 112 --- .../commandhandler/CommandDispatcher.java | 5 +- .../commandhandler/CommandHandler.java | 8 +- .../commandhandler/ContainerReportHandler.java | 114 --- .../DeleteBlocksCommandHandler.java | 21 +- .../states/datanode/RunningDatanodeState.java | 1 + .../states/endpoint/HeartbeatEndpointTask.java | 38 +- .../states/endpoint/RegisterEndpointTask.java | 42 +- .../transport/server/GrpcXceiverService.java | 82 ++ .../common/transport/server/XceiverServer.java | 18 +- .../transport/server/XceiverServerGrpc.java | 106 +++ .../transport/server/XceiverServerHandler.java | 8 +- .../server/XceiverServerInitializer.java | 18 +- .../server/ratis/ContainerStateMachine.java | 46 +- .../server/ratis/XceiverServerRatis.java | 36 +- .../container/common/utils/ContainerCache.java | 36 +- .../container/ozoneimpl/OzoneContainer.java | 34 +- .../StorageContainerDatanodeProtocol.java | 38 +- .../protocol/StorageContainerNodeProtocol.java | 16 +- .../commands/CloseContainerCommand.java | 32 +- .../protocol/commands/DeleteBlocksCommand.java | 18 +- .../protocol/commands/RegisteredCommand.java | 26 +- .../protocol/commands/ReregisterCommand.java | 16 +- .../ozone/protocol/commands/SCMCommand.java | 4 +- .../protocol/commands/SendContainerCommand.java | 80 -- ...rDatanodeProtocolClientSideTranslatorPB.java | 58 +- ...rDatanodeProtocolServerSideTranslatorPB.java | 56 +- .../StorageContainerDatanodeProtocol.proto | 294 +++---- .../ozone/container/common/ScmTestMock.java | 113 ++- .../common/TestDatanodeStateMachine.java | 23 +- hadoop-hdds/framework/pom.xml | 4 +- .../apache/hadoop/hdds/server/events/Event.java | 42 + .../hdds/server/events/EventExecutor.java | 68 ++ .../hadoop/hdds/server/events/EventHandler.java | 33 + .../hdds/server/events/EventPublisher.java | 28 + .../hadoop/hdds/server/events/EventQueue.java | 213 +++++ .../server/events/SingleThreadExecutor.java | 103 +++ .../hadoop/hdds/server/events/TypedEvent.java | 51 ++ .../hadoop/hdds/server/events/package-info.java | 23 + .../hdds/server/events/TestEventQueue.java | 113 +++ .../hdds/server/events/TestEventQueueChain.java | 79 ++ hadoop-hdds/pom.xml | 4 +- hadoop-hdds/server-scm/pom.xml | 4 +- .../hadoop/hdds/scm/block/BlockManager.java | 12 +- .../hadoop/hdds/scm/block/BlockManagerImpl.java | 172 +--- .../block/DatanodeDeletedBlockTransactions.java | 6 +- .../hadoop/hdds/scm/block/DeletedBlockLog.java | 6 +- .../hdds/scm/block/DeletedBlockLogImpl.java | 24 +- .../hdds/scm/container/ContainerMapping.java | 92 +-- .../scm/container/ContainerStateManager.java | 36 +- .../hadoop/hdds/scm/container/Mapping.java | 42 +- .../scm/container/closer/ContainerCloser.java | 16 +- .../replication/ContainerSupervisor.java | 13 +- .../container/replication/InProgressPool.java | 82 +- .../hdds/scm/container/states/package-info.java | 2 +- .../hdds/scm/exceptions/SCMException.java | 4 +- .../hdds/scm/node/HeartbeatQueueItem.java | 35 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 88 +- .../hdds/scm/node/SCMNodePoolManager.java | 2 +- .../hdds/scm/node/SCMNodeStorageStatMXBean.java | 77 ++ .../hdds/scm/node/SCMNodeStorageStatMap.java | 368 +++++++++ .../hdds/scm/node/StorageReportResult.java | 87 ++ .../hdds/scm/node/states/Node2ContainerMap.java | 184 +++++ .../hdds/scm/node/states/ReportResult.java | 86 ++ .../hdds/scm/node/states/package-info.java | 22 + .../hdds/scm/pipelines/PipelineManager.java | 15 +- .../hdds/scm/pipelines/PipelineSelector.java | 9 +- .../scm/pipelines/ratis/RatisManagerImpl.java | 2 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 20 +- .../scm/server/SCMClientProtocolServer.java | 58 +- .../scm/server/SCMDatanodeProtocolServer.java | 207 +++-- .../scm/server/StorageContainerManager.java | 17 +- .../TestStorageContainerManagerHttpServer.java | 1 - .../org/apache/hadoop/hdds/scm/TestUtils.java | 55 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 34 +- .../hdds/scm/block/TestDeletedBlockLog.java | 78 +- .../hadoop/hdds/scm/block/package-info.java | 23 + .../hdds/scm/container/MockNodeManager.java | 27 +- .../scm/container/TestContainerMapping.java | 111 +-- .../container/closer/TestContainerCloser.java | 40 +- .../hdds/scm/container/closer/package-info.java | 22 + .../hadoop/hdds/scm/container/package-info.java | 22 + .../hdds/scm/container/states/package-info.java | 22 + .../hdds/scm/node/TestContainerPlacement.java | 33 +- .../hadoop/hdds/scm/node/TestNodeManager.java | 218 +++-- .../scm/node/TestSCMNodeStorageStatMap.java | 263 ++++++ .../hadoop/hdds/scm/node/package-info.java | 22 + .../scm/node/states/Node2ContainerMapTest.java | 308 +++++++ .../hdds/scm/node/states/package-info.java | 23 + .../ozone/container/common/TestEndPoint.java | 159 +--- .../ozone/container/common/package-info.java | 22 + .../ozone/container/placement/package-info.java | 22 + .../replication/TestContainerSupervisor.java | 272 ------ .../ReplicationDatanodeStateManager.java | 100 --- .../testutils/ReplicationNodeManagerMock.java | 16 +- hadoop-hdds/tools/pom.xml | 4 +- .../cli/container/CloseContainerHandler.java | 26 +- .../cli/container/ContainerCommandHandler.java | 1 - .../cli/container/CreateContainerHandler.java | 30 +- .../cli/container/DeleteContainerHandler.java | 26 +- .../scm/cli/container/InfoContainerHandler.java | 42 +- .../scm/cli/container/ListContainerHandler.java | 20 +- .../dev-support/findbugsExcludeFile.xml | 5 + .../java/org/apache/hadoop/hdfs/DFSClient.java | 10 + .../org/apache/hadoop/hdfs/DFSInputStream.java | 17 +- .../hadoop/hdfs/DFSStripedInputStream.java | 20 +- .../hadoop/hdfs/DistributedFileSystem.java | 10 + .../hadoop/hdfs/protocol/ClientProtocol.java | 9 + .../apache/hadoop/hdfs/protocol/DatanodeID.java | 50 +- .../hadoop/hdfs/protocol/HdfsConstants.java | 7 + .../ClientNamenodeProtocolTranslatorPB.java | 17 + .../hadoop/hdfs/protocolPB/PBHelperClient.java | 67 +- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 16 + .../hadoop/hdfs/web/WebHdfsFileSystem.java | 11 +- .../src/main/proto/ClientNamenodeProtocol.proto | 9 + .../org/apache/hadoop/test/TestHdfsHelper.java | 4 +- .../federation/router/RouterAdminServer.java | 28 +- .../federation/router/RouterRpcServer.java | 7 + .../federation/router/TestRouterAdminCLI.java | 10 + .../federation/router/TestRouterQuota.java | 43 +- .../jdiff/Apache_Hadoop_HDFS_2.9.1.xml | 312 +++++++ ...tNamenodeProtocolServerSideTranslatorPB.java | 17 + .../hadoop/hdfs/qjournal/server/Journal.java | 115 +-- .../BlockPlacementPolicyDefault.java | 13 +- .../server/datanode/web/DatanodeHttpServer.java | 14 +- .../hdfs/server/namenode/FSDirAttrOp.java | 4 +- .../hdfs/server/namenode/FSEditLogLoader.java | 23 +- .../hadoop/hdfs/server/namenode/FSImage.java | 16 +- .../hdfs/server/namenode/FSNamesystem.java | 12 + .../hdfs/server/namenode/INodesInPath.java | 2 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 6 + .../hdfs/server/namenode/NameNodeUtils.java | 2 +- .../hdfs/server/namenode/ha/EditLogTailer.java | 27 +- .../snapshot/DirectorySnapshottableFeature.java | 6 +- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 99 +++ .../apache/hadoop/hdfs/tools/DebugAdmin.java | 30 +- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 19 + .../src/main/resources/hdfs-default.xml | 3 + .../src/main/webapps/datanode/datanode.html | 2 +- .../src/site/markdown/ArchivalStorage.md | 4 +- .../src/site/markdown/HDFSCommands.md | 2 + .../markdown/HDFSHighAvailabilityWithQJM.md | 2 + .../hadoop/fs/TestResolveHdfsSymlink.java | 2 +- .../apache/hadoop/hdfs/TestDFSOutputStream.java | 2 +- .../org/apache/hadoop/hdfs/TestDFSShell.java | 10 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 70 ++ .../hadoop/hdfs/TestDatanodeRegistration.java | 149 ++-- .../apache/hadoop/hdfs/TestEncryptionZones.java | 3 - .../hdfs/TestErasureCodingExerciseAPIs.java | 5 +- .../hadoop/hdfs/TestHDFSFileSystemContract.java | 6 +- .../apache/hadoop/hdfs/TestHDFSServerPorts.java | 5 + .../apache/hadoop/hdfs/TestMiniDFSCluster.java | 44 +- .../java/org/apache/hadoop/hdfs/TestQuota.java | 48 +- .../hadoop/hdfs/TestSnapshotCommands.java | 5 + .../client/impl/TestBlockReaderLocalLegacy.java | 7 +- .../hdfs/qjournal/MiniJournalCluster.java | 5 + .../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 10 +- .../client/TestQuorumJournalManager.java | 3 +- .../blockmanagement/TestBlockManager.java | 155 ++-- .../TestNameNodePrunesMissingStorages.java | 97 ++- .../hdfs/server/datanode/TestBlockScanner.java | 5 +- .../datanode/TestDataNodeFaultInjector.java | 2 +- .../TestDataXceiverBackwardsCompat.java | 10 + .../server/datanode/TestDirectoryScanner.java | 3 +- .../fsdataset/impl/LazyPersistTestCase.java | 3 +- .../fsdataset/impl/TestFsDatasetImpl.java | 8 +- .../datanode/web/TestDatanodeHttpXFrame.java | 30 +- .../diskbalancer/DiskBalancerTestUtil.java | 5 +- .../command/TestDiskBalancerCommand.java | 15 +- .../namenode/TestAuditLoggerWithCommands.java | 5 +- .../server/namenode/TestNameNodeMXBean.java | 7 +- .../namenode/TestNestedEncryptionZones.java | 9 + .../server/namenode/TestQuotaByStorageType.java | 11 +- .../server/namenode/ha/TestEditLogTailer.java | 13 +- .../namenode/metrics/TestNameNodeMetrics.java | 8 +- .../namenode/snapshot/TestSnapshotRename.java | 16 + .../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 65 +- .../hadoop/hdfs/tools/TestDFSAdminWithHA.java | 71 +- .../hadoop/hdfs/tools/TestDebugAdmin.java | 2 +- .../apache/hadoop/hdfs/web/TestJsonUtil.java | 48 +- .../apache/hadoop/hdfs/web/TestWebHdfsUrl.java | 55 ++ .../apache/hadoop/security/TestPermission.java | 13 +- .../v2/app/job/impl/TaskAttemptImpl.java | 2 +- .../mapreduce/v2/app/webapp/AppController.java | 2 +- .../mapreduce/v2/app/webapp/TaskPage.java | 10 +- .../mapreduce/v2/app/webapp/TasksBlock.java | 6 +- .../v2/app/rm/ContainerRequestCreator.java | 57 ++ .../v2/app/rm/TestRMContainerAllocator.java | 534 ++++++------ .../v2/app/webapp/TestAppController.java | 2 +- .../mapred/LocalDistributedCacheManager.java | 31 +- .../apache/hadoop/mapred/LocalJobRunner.java | 14 +- .../mapreduce/checkpoint/RandomNameCNS.java | 2 +- .../HumanReadableHistoryViewerPrinter.java | 2 +- .../lib/output/PathOutputCommitterFactory.java | 2 +- .../hadoop/mapreduce/security/TokenCache.java | 2 +- .../org/apache/hadoop/mapreduce/tools/CLI.java | 2 +- .../lib/partition/TestRehashPartitioner.java | 2 +- .../mapreduce/v2/hs/webapp/HsJobBlock.java | 18 +- .../mapreduce/v2/hs/webapp/HsJobsBlock.java | 8 +- .../mapreduce/v2/hs/webapp/HsTaskPage.java | 10 +- .../mapreduce/v2/hs/webapp/TestHsJobBlock.java | 20 +- .../org/apache/hadoop/mapred/ClientCache.java | 2 +- .../hadoop/mapred/ClientServiceDelegate.java | 2 +- .../org/apache/hadoop/mapred/NotRunningJob.java | 12 +- .../org/apache/hadoop/mapred/YARNRunner.java | 2 +- .../apache/hadoop/mapred/TestJobCounters.java | 2 +- .../mapreduce/lib/db/TestDBOutputFormat.java | 2 +- .../TestMapReduceJobControlWithMocks.java | 19 +- .../hadoop/mapred/FadvisedChunkedFile.java | 57 +- .../hadoop/mapred/TestFadvisedChunkedFile.java | 55 ++ .../hadoop-mapreduce-client/pom.xml | 5 - hadoop-ozone/acceptance-test/README.md | 24 +- .../dev-support/bin/robot-all.sh | 18 + .../acceptance-test/dev-support/bin/robot.sh | 38 + hadoop-ozone/acceptance-test/pom.xml | 33 +- .../acceptance-test/src/test/compose/.env | 2 +- .../src/test/compose/docker-compose.yaml | 9 +- .../src/test/compose/docker-config | 1 + .../robotframework/acceptance/ozone-shell.robot | 256 ++++++ .../test/robotframework/acceptance/ozone.robot | 20 +- .../apache/hadoop/ozone/client/ObjectStore.java | 58 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 51 +- .../ozone/client/OzoneClientException.java | 54 ++ .../hadoop/ozone/client/OzoneClientFactory.java | 2 +- .../hadoop/ozone/client/OzoneClientUtils.java | 87 ++ .../apache/hadoop/ozone/client/OzoneVolume.java | 44 +- .../ozone/client/io/ChunkGroupInputStream.java | 27 +- .../ozone/client/io/ChunkGroupOutputStream.java | 37 +- .../client/io/OzoneContainerTranslation.java | 13 +- .../ozone/client/protocol/ClientProtocol.java | 10 + .../hadoop/ozone/client/rest/RestClient.java | 72 +- .../hadoop/ozone/client/rpc/RpcClient.java | 53 +- .../hadoop/ozone/web/client/OzoneBucket.java | 645 --------------- .../hadoop/ozone/web/client/OzoneKey.java | 44 - .../ozone/web/client/OzoneRestClient.java | 803 ------------------ .../web/client/OzoneRestClientException.java | 54 -- .../hadoop/ozone/web/client/OzoneVolume.java | 583 ------------- .../hadoop/ozone/web/client/package-info.java | 34 - hadoop-ozone/common/pom.xml | 2 +- hadoop-ozone/common/src/main/bin/ozone | 4 + hadoop-ozone/common/src/main/bin/start-ozone.sh | 116 ++- .../ozone/client/rest/headers/Header.java | 2 + .../hadoop/ozone/ksm/helpers/KsmKeyInfo.java | 42 +- .../ozone/ksm/helpers/KsmKeyLocationInfo.java | 41 +- .../ksm/helpers/KsmKeyLocationInfoGroup.java | 2 +- .../ksm/protocol/KeySpaceManagerProtocol.java | 9 +- ...ceManagerProtocolClientSideTranslatorPB.java | 29 +- .../hadoop/ozone/web/handlers/UserArgs.java | 8 +- .../main/proto/KeySpaceManagerProtocol.proto | 41 +- hadoop-ozone/docs/README.md | 52 ++ hadoop-ozone/docs/archetypes/default.md | 17 + hadoop-ozone/docs/config.toml | 23 + hadoop-ozone/docs/content/CommandShell.md | 153 ++++ hadoop-ozone/docs/content/GettingStarted.md | 352 ++++++++ hadoop-ozone/docs/content/Metrics.md | 170 ++++ hadoop-ozone/docs/content/Rest.md | 553 +++++++++++++ hadoop-ozone/docs/content/_index.md | 102 +++ .../docs/dev-support/bin/generate-site.sh | 29 + hadoop-ozone/docs/pom.xml | 58 ++ hadoop-ozone/docs/static/OzoneOverview.png | Bin 0 -> 41729 bytes hadoop-ozone/docs/static/OzoneOverview.svg | 225 +++++ hadoop-ozone/docs/static/SCMBlockDiagram.png | Bin 0 -> 14714 bytes .../ozonedoc/layouts/_default/single.html | 32 + .../docs/themes/ozonedoc/layouts/index.html | 21 + .../ozonedoc/layouts/partials/footer.html | 19 + .../ozonedoc/layouts/partials/header.html | 31 + .../ozonedoc/layouts/partials/navbar.html | 33 + .../ozonedoc/layouts/partials/sidebar.html | 43 + .../ozonedoc/static/css/bootstrap-theme.min.css | 6 + .../static/css/bootstrap-theme.min.css.map | 1 + .../ozonedoc/static/css/bootstrap.min.css | 6 + .../ozonedoc/static/css/bootstrap.min.css.map | 1 + .../themes/ozonedoc/static/css/ozonedoc.css | 128 +++ .../fonts/glyphicons-halflings-regular.eot | Bin 0 -> 20127 bytes .../fonts/glyphicons-halflings-regular.svg | 288 +++++++ .../fonts/glyphicons-halflings-regular.ttf | Bin 0 -> 45404 bytes .../fonts/glyphicons-halflings-regular.woff | Bin 0 -> 23424 bytes .../fonts/glyphicons-halflings-regular.woff2 | Bin 0 -> 18028 bytes .../themes/ozonedoc/static/js/bootstrap.min.js | 7 + .../themes/ozonedoc/static/js/jquery.min.js | 5 + .../docs/themes/ozonedoc/static/js/ozonedoc.js | 23 + hadoop-ozone/docs/themes/ozonedoc/theme.toml | 2 + hadoop-ozone/integration-test/pom.xml | 4 +- .../container/TestContainerStateManager.java | 124 +-- .../hadoop/ozone/MiniOzoneClusterImpl.java | 22 +- .../apache/hadoop/ozone/RatisTestHelper.java | 17 +- .../hadoop/ozone/TestContainerOperations.java | 10 +- .../hadoop/ozone/TestMiniOzoneCluster.java | 11 +- .../ozone/TestStorageContainerManager.java | 96 +-- .../TestStorageContainerManagerHelper.java | 36 +- .../ozone/client/rest/TestOzoneRestClient.java | 30 + .../ozone/client/rpc/TestOzoneRpcClient.java | 54 +- .../ozone/container/ContainerTestHelper.java | 202 ++--- .../common/TestBlockDeletingService.java | 27 +- .../TestContainerDeletionChoosingPolicy.java | 28 +- .../common/impl/TestContainerPersistence.java | 440 +++++----- .../TestCloseContainerHandler.java | 14 +- .../container/metrics/TestContainerMetrics.java | 25 +- .../container/ozoneimpl/TestOzoneContainer.java | 154 ++-- .../ozoneimpl/TestOzoneContainerRatis.java | 3 +- .../container/server/TestContainerServer.java | 36 +- ...TestGenerateOzoneRequiredConfigurations.java | 100 +++ .../ozone/ksm/TestContainerReportWithKeys.java | 8 +- .../hadoop/ozone/ksm/TestKeySpaceManager.java | 132 ++- .../ksm/TestKeySpaceManagerRestInterface.java | 5 +- .../ozone/ksm/TestKsmBlockVersioning.java | 2 +- .../hadoop/ozone/ozShell/TestOzoneShell.java | 298 +++++-- .../hadoop/ozone/scm/TestAllocateContainer.java | 24 +- .../hadoop/ozone/scm/TestContainerSQLCli.java | 63 +- .../ozone/scm/TestContainerSmallFile.java | 70 +- .../org/apache/hadoop/ozone/scm/TestSCMCli.java | 258 +++--- .../apache/hadoop/ozone/scm/TestSCMMetrics.java | 73 +- .../ozone/scm/TestXceiverClientManager.java | 145 ++-- .../ozone/scm/TestXceiverClientMetrics.java | 29 +- .../ozone/web/TestDistributedOzoneVolumes.java | 4 +- .../hadoop/ozone/web/TestLocalOzoneVolumes.java | 4 +- .../ozone/web/TestOzoneRestWithMiniCluster.java | 207 ++--- .../hadoop/ozone/web/TestOzoneWebAccess.java | 4 +- .../hadoop/ozone/web/client/TestBuckets.java | 192 +++-- .../ozone/web/client/TestBucketsRatis.java | 15 +- .../hadoop/ozone/web/client/TestKeys.java | 293 ++++--- .../hadoop/ozone/web/client/TestKeysRatis.java | 29 +- .../ozone/web/client/TestOzoneClient.java | 4 +- .../hadoop/ozone/web/client/TestVolume.java | 286 +++---- .../ozone/web/client/TestVolumeRatis.java | 33 +- .../ozone/web/OzoneHddsDatanodeService.java | 5 +- .../hadoop/ozone/web/handlers/KeyHandler.java | 45 + .../hadoop/ozone/web/interfaces/Keys.java | 31 + .../ozone/web/interfaces/StorageHandler.java | 9 + .../web/localstorage/LocalStorageHandler.java | 6 + .../web/storage/DistributedStorageHandler.java | 11 + hadoop-ozone/ozone-manager/pom.xml | 4 +- .../ozone/ksm/KSMMetadataManagerImpl.java | 10 +- .../org/apache/hadoop/ozone/ksm/KSMMetrics.java | 22 + .../hadoop/ozone/ksm/KeyDeletingService.java | 3 +- .../org/apache/hadoop/ozone/ksm/KeyManager.java | 10 + .../apache/hadoop/ozone/ksm/KeyManagerImpl.java | 90 +- .../hadoop/ozone/ksm/KeySpaceManager.java | 17 +- .../hadoop/ozone/ksm/OpenKeyCleanupService.java | 3 +- .../ozone/ksm/exceptions/KSMException.java | 2 + ...ceManagerProtocolServerSideTranslatorPB.java | 32 +- .../hadoop/ozone/web/ozShell/Handler.java | 102 ++- .../web/ozShell/bucket/CreateBucketHandler.java | 32 +- .../web/ozShell/bucket/DeleteBucketHandler.java | 20 +- .../web/ozShell/bucket/InfoBucketHandler.java | 25 +- .../web/ozShell/bucket/ListBucketHandler.java | 51 +- .../web/ozShell/bucket/UpdateBucketHandler.java | 41 +- .../web/ozShell/keys/DeleteKeyHandler.java | 26 +- .../ozone/web/ozShell/keys/GetKeyHandler.java | 46 +- .../ozone/web/ozShell/keys/InfoKeyHandler.java | 31 +- .../ozone/web/ozShell/keys/ListKeyHandler.java | 48 +- .../ozone/web/ozShell/keys/PutKeyHandler.java | 49 +- .../web/ozShell/volume/CreateVolumeHandler.java | 28 +- .../web/ozShell/volume/DeleteVolumeHandler.java | 18 +- .../web/ozShell/volume/InfoVolumeHandler.java | 38 +- .../web/ozShell/volume/ListVolumeHandler.java | 58 +- .../web/ozShell/volume/UpdateVolumeHandler.java | 24 +- .../src/main/site/markdown/OzoneCommandShell.md | 150 ---- .../site/markdown/OzoneGettingStarted.md.vm | 347 -------- .../src/main/site/markdown/OzoneMetrics.md | 166 ---- .../src/main/site/markdown/OzoneOverview.md | 88 -- .../src/main/site/markdown/OzoneRest.md | 549 ------------- hadoop-ozone/pom.xml | 3 +- hadoop-ozone/tools/pom.xml | 8 - .../GenerateOzoneRequiredConfigurations.java | 174 ++++ .../hadoop/ozone/genconf/package-info.java | 24 + .../genesis/BenchMarkContainerStateMap.java | 10 +- .../genesis/BenchMarkDatanodeDispatcher.java | 198 +++-- .../ozone/genesis/BenchMarkRocksDbStore.java | 14 +- .../ozone/genesis/GenesisMemoryProfiler.java | 4 +- .../hadoop/ozone/genesis/GenesisUtil.java | 12 +- .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 79 +- hadoop-project-dist/pom.xml | 2 +- hadoop-project/pom.xml | 30 +- hadoop-tools/hadoop-aliyun/pom.xml | 2 +- .../hadoop/tools/TestHadoopArchiveLogs.java | 12 +- .../hadoop/fs/s3a/S3AInstrumentation.java | 3 - .../staging/DirectoryStagingCommitter.java | 7 +- .../fs/s3a/s3guard/LocalMetadataStore.java | 109 ++- .../hadoop/fs/s3a/s3guard/LruHashMap.java | 50 -- .../apache/hadoop/fs/s3a/ITestS3AMetrics.java | 5 - .../TestStagingDirectoryOutputCommitter.java | 8 +- .../s3guard/AbstractS3GuardToolTestBase.java | 157 +++- .../s3a/s3guard/ITestS3GuardToolDynamoDB.java | 5 - .../fs/s3a/s3guard/ITestS3GuardToolLocal.java | 73 -- .../fs/s3a/s3guard/TestLocalMetadataStore.java | 121 ++- .../org/apache/hadoop/fs/adl/AdlConfKeys.java | 1 + .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 13 + .../src/site/markdown/troubleshooting_adl.md | 9 + .../fs/adl/live/AdlStorageConfiguration.java | 12 +- .../fs/adl/live/TestAdlSdkConfiguration.java | 77 ++ hadoop-tools/hadoop-azure/pom.xml | 2 +- .../hadoop/fs/azure/PageBlobInputStream.java | 123 ++- .../hadoop/fs/azure/PageBlobOutputStream.java | 13 +- .../fs/azure/SyncableDataOutputStream.java | 4 - .../fs/azure/ITestOutputStreamSemantics.java | 385 +++++++++ .../fs/azure/ITestPageBlobInputStream.java | 527 ++++++++++++ hadoop-tools/hadoop-ozone/pom.xml | 25 - .../org/apache/hadoop/fs/ozone/Constants.java | 6 - .../java/org/apache/hadoop/fs/ozone/OzFs.java | 3 +- .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 33 +- .../hadoop/fs/ozone/TestOzoneFSInputStream.java | 6 +- .../fs/ozone/TestOzoneFileInterfaces.java | 5 +- .../hadoop/fs/ozone/contract/OzoneContract.java | 6 +- .../org/apache/hadoop/streaming/StreamJob.java | 4 +- .../hadoop-yarn/conf/container-executor.cfg | 1 + .../hadoop/yarn/api/ApplicationConstants.java | 9 +- .../records/timelineservice/TimelineDomain.java | 178 ++++ .../hadoop/yarn/conf/YarnConfiguration.java | 57 +- .../hadoop/yarn/util/UnitsConversionUtil.java | 44 +- .../yarn/conf/TestYarnConfigurationFields.java | 2 + .../distributedshell/ApplicationMaster.java | 98 ++- .../distributedshell/TestDistributedShell.java | 10 + .../dev-support/findbugs-exclude.xml | 20 - .../hadoop-yarn-services-api/pom.xml | 144 ---- .../yarn/service/client/ApiServiceClient.java | 553 ------------- .../client/SystemServiceManagerImpl.java | 391 --------- .../yarn/service/client/package-info.java | 28 - .../hadoop/yarn/service/webapp/ApiServer.java | 750 ----------------- .../yarn/service/webapp/ApiServerWebApp.java | 161 ---- .../yarn/service/webapp/package-info.java | 28 - .../definition/YARN-Services-Examples.md | 445 ---------- ...RN-Simplified-V1-API-Layer-For-Services.yaml | 586 ------------- .../src/main/resources/log4j-server.properties | 76 -- .../src/main/resources/webapps/api-server/app | 16 - .../src/main/webapp/WEB-INF/web.xml | 36 - .../hadoop/yarn/service/ServiceClientTest.java | 193 ----- .../hadoop/yarn/service/TestApiServer.java | 558 ------------- .../service/client/TestApiServiceClient.java | 302 ------- .../client/TestSystemServiceManagerImpl.java | 182 ---- .../src/test/resources/example-app.json | 16 - .../src/test/resources/log4j.properties | 19 - .../resources/system-services/bad/bad.yarnfile | 16 - .../sync/user1/example-app1.yarnfile | 16 - .../sync/user1/example-app2.yarnfile | 16 - .../sync/user1/example-app3.json | 16 - .../sync/user2/example-app1.yarnfile | 16 - .../sync/user2/example-app2.yarnfile | 16 - .../dev-support/findbugs-exclude.xml | 20 + .../hadoop-yarn-services-api/pom.xml | 144 ++++ .../yarn/service/client/ApiServiceClient.java | 598 ++++++++++++++ .../client/SystemServiceManagerImpl.java | 391 +++++++++ .../yarn/service/client/package-info.java | 28 + .../hadoop/yarn/service/webapp/ApiServer.java | 822 +++++++++++++++++++ .../yarn/service/webapp/ApiServerWebApp.java | 161 ++++ .../yarn/service/webapp/package-info.java | 28 + .../definition/YARN-Services-Examples.md | 444 ++++++++++ ...RN-Simplified-V1-API-Layer-For-Services.yaml | 594 ++++++++++++++ .../src/main/resources/log4j-server.properties | 76 ++ .../src/main/resources/webapps/api-server/app | 16 + .../src/main/webapp/WEB-INF/web.xml | 36 + .../hadoop/yarn/service/ServiceClientTest.java | 226 +++++ .../hadoop/yarn/service/TestApiServer.java | 623 ++++++++++++++ .../service/client/TestApiServiceClient.java | 314 +++++++ .../client/TestSystemServiceManagerImpl.java | 182 ++++ .../src/test/resources/example-app.json | 16 + .../src/test/resources/log4j.properties | 19 + .../resources/system-services/bad/bad.yarnfile | 16 + .../sync/user1/example-app1.yarnfile | 16 + .../sync/user1/example-app2.yarnfile | 16 + .../sync/user1/example-app3.json | 16 + .../sync/user2/example-app1.yarnfile | 16 + .../sync/user2/example-app2.yarnfile | 16 + .../hadoop/yarn/service/ClientAMService.java | 7 +- .../hadoop/yarn/service/ServiceContext.java | 4 + .../hadoop/yarn/service/ServiceManager.java | 30 +- .../hadoop/yarn/service/ServiceMaster.java | 95 ++- .../hadoop/yarn/service/ServiceScheduler.java | 73 +- .../yarn/service/UpgradeComponentsFinder.java | 104 +-- .../yarn/service/api/records/Component.java | 71 ++ .../service/api/records/ComponentState.java | 2 +- .../yarn/service/client/ServiceClient.java | 71 +- .../service/component/AlwaysRestartPolicy.java | 82 ++ .../yarn/service/component/Component.java | 331 +++++--- .../component/ComponentRestartPolicy.java | 45 + .../service/component/NeverRestartPolicy.java | 82 ++ .../component/OnFailureRestartPolicy.java | 87 ++ .../component/instance/ComponentInstance.java | 145 +++- .../yarn/service/conf/RestApiConstants.java | 2 + .../containerlaunch/AbstractLauncher.java | 26 +- .../exceptions/RestApiErrorMessages.java | 16 + .../provider/AbstractProviderService.java | 91 +- .../yarn/service/provider/ProviderUtils.java | 3 +- .../provider/docker/DockerProviderService.java | 42 + .../yarn/service/utils/ServiceApiUtil.java | 107 ++- .../hadoop/yarn/service/utils/ServiceUtils.java | 18 + .../hadoop/yarn/service/MockServiceAM.java | 17 +- .../hadoop/yarn/service/ServiceTestUtils.java | 46 +- .../TestDefaultUpgradeComponentsFinder.java | 30 +- .../hadoop/yarn/service/TestServiceAM.java | 42 + .../hadoop/yarn/service/TestServiceApiUtil.java | 85 +- .../hadoop/yarn/service/TestServiceManager.java | 6 +- .../yarn/service/TestYarnNativeServices.java | 40 +- .../yarn/service/client/TestServiceCLI.java | 18 + .../yarn/service/component/TestComponent.java | 99 ++- .../component/TestComponentRestartPolicy.java | 130 +++ .../instance/TestComponentInstance.java | 484 ++++++++++- .../containerlaunch/TestAbstractLauncher.java | 54 ++ .../service/provider/TestProviderUtils.java | 6 +- .../hadoop-yarn-services/pom.xml | 1 + .../hadoop-yarn-applications/pom.xml | 1 - .../hadoop/yarn/client/api/AppAdminClient.java | 12 + .../yarn/client/api/impl/AMRMClientImpl.java | 151 +--- .../yarn/client/api/impl/YarnClientImpl.java | 2 +- .../hadoop/yarn/client/cli/ApplicationCLI.java | 17 +- .../hadoop/yarn/client/cli/SchedConfCLI.java | 42 +- .../yarn/client/api/impl/TestYarnClient.java | 4 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 3 + .../hadoop-yarn/hadoop-yarn-common/pom.xml | 6 +- .../hadoop/yarn/client/AMRMClientUtils.java | 262 ++++++ .../logaggregation/AggregatedLogFormat.java | 14 +- .../LogAggregationDFSException.java | 45 + .../LogAggregationFileController.java | 21 +- .../tfile/LogAggregationTFileController.java | 13 +- .../yarn/security/AMRMTokenIdentifier.java | 33 +- .../yarn/security/ContainerTokenIdentifier.java | 102 ++- .../hadoop/yarn/security/NMTokenIdentifier.java | 32 +- .../java/org/apache/hadoop/yarn/util/Apps.java | 33 +- .../yarn/util/DockerClientConfigHandler.java | 23 +- .../resource/DefaultResourceCalculator.java | 15 +- .../resource/DominantResourceCalculator.java | 39 +- .../yarn/util/resource/ResourceCalculator.java | 13 +- .../hadoop/yarn/util/resource/Resources.java | 5 - .../apache/hadoop/yarn/webapp/Controller.java | 4 + .../org/apache/hadoop/yarn/webapp/View.java | 24 +- .../yarn/webapp/util/YarnWebServiceUtils.java | 17 +- .../src/main/resources/yarn-default.xml | 103 ++- .../logaggregation/TestAggregatedLogFormat.java | 19 +- .../logaggregation/TestContainerLogsUtils.java | 4 +- .../resourcetypes/ResourceTypesTestHelper.java | 93 +++ .../security/TestDockerClientConfigHandler.java | 4 +- .../yarn/security/TestYARNTokenIdentifier.java | 146 +++- .../server/dns/BaseServiceRecordProcessor.java | 20 + .../dns/ContainerServiceRecordProcessor.java | 3 +- .../registry/server/dns/TestRegistryDNS.java | 62 +- .../pom.xml | 5 + .../webapp/AHSWebServices.java | 18 +- .../hadoop/yarn/server/AMRMClientRelayer.java | 364 ++++++++ .../hadoop/yarn/server/api/ServerRMProxy.java | 11 +- .../server/api/protocolrecords/RemoteNode.java | 40 +- .../impl/pb/RemoteNodePBImpl.java | 19 + .../failover/FederationProxyProviderUtil.java | 2 +- .../metrics/AppAttemptMetricsConstants.java | 5 + .../apache/hadoop/yarn/server/package-info.java | 18 + .../OpportunisticContainerAllocator.java | 38 +- .../server/scheduler/ResourceRequestSet.java | 206 +++++ .../server/scheduler/ResourceRequestSetKey.java | 133 +++ .../server/scheduler/SchedulerRequestKey.java | 4 +- .../yarn/server/uam/UnmanagedAMPoolManager.java | 2 +- .../server/uam/UnmanagedApplicationManager.java | 2 +- .../yarn/server/utils/AMRMClientUtils.java | 191 ----- .../hadoop/yarn/server/utils/BuilderUtils.java | 8 +- .../hadoop/yarn/server/webapp/WebServices.java | 2 +- .../yarn_server_common_service_protos.proto | 1 + .../yarn/server/MockResourceManagerFacade.java | 2 +- .../yarn/server/TestAMRMClientRelayer.java | 275 +++++++ .../yarn/server/api/TestServerRMProxy.java | 52 ++ .../TestOpportunisticContainerAllocator.java | 37 + .../src/CMakeLists.txt | 19 + .../amrmproxy/FederationInterceptor.java | 2 +- .../container/ContainerImpl.java | 4 +- .../container/SlidingWindowRetryPolicy.java | 63 +- .../launcher/ContainerLaunch.java | 29 +- .../CGroupElasticMemoryController.java | 476 +++++++++++ .../CGroupsBlkioResourceHandlerImpl.java | 6 + .../CGroupsCpuResourceHandlerImpl.java | 75 +- .../linux/resources/CGroupsHandler.java | 6 + .../linux/resources/CGroupsHandlerImpl.java | 6 +- .../CGroupsMemoryResourceHandlerImpl.java | 88 +- .../linux/resources/DefaultOOMHandler.java | 254 ++++++ .../NetworkPacketTaggingHandlerImpl.java | 6 + .../linux/resources/ResourceHandler.java | 16 +- .../linux/resources/ResourceHandlerChain.java | 18 + .../TrafficControlBandwidthHandlerImpl.java | 6 + .../resources/fpga/FpgaResourceHandlerImpl.java | 6 + .../resources/gpu/GpuResourceHandlerImpl.java | 6 + .../resources/numa/NumaResourceHandlerImpl.java | 6 + .../runtime/DockerLinuxContainerRuntime.java | 200 +++-- .../linux/runtime/docker/DockerClient.java | 126 +-- .../linux/runtime/docker/DockerRunCommand.java | 44 + .../logaggregation/AppLogAggregatorImpl.java | 49 +- .../monitor/ContainersMonitorImpl.java | 50 ++ .../scheduler/ContainerScheduler.java | 30 + .../executor/ContainerSignalContext.java | 41 + .../timelineservice/NMTimelineEvent.java | 12 +- .../timelineservice/NMTimelineEventType.java | 3 + .../timelineservice/NMTimelinePublisher.java | 23 +- .../nodemanager/webapp/NMWebServices.java | 63 +- .../impl/container-executor.c | 224 +++-- .../impl/container-executor.h | 5 + .../container-executor/impl/utils/docker-util.c | 655 +++++++-------- .../container-executor/impl/utils/docker-util.h | 52 +- .../impl/utils/string-utils.c | 24 + .../impl/utils/string-utils.h | 4 + .../test/utils/test_docker_util.cc | 577 ++++++++----- .../native/oom-listener/impl/oom_listener.c | 171 ++++ .../native/oom-listener/impl/oom_listener.h | 102 +++ .../oom-listener/impl/oom_listener_main.c | 104 +++ .../oom-listener/test/oom_listener_test_main.cc | 292 +++++++ .../nodemanager/TestNodeManagerResync.java | 87 +- .../containermanager/TestContainerManager.java | 20 +- .../container/TestSlidingWindowRetryPolicy.java | 6 + .../launcher/TestContainerLaunch.java | 94 ++- .../resources/DummyRunnableWithContext.java | 31 + .../TestCGroupElasticMemoryController.java | 319 +++++++ .../TestCGroupsCpuResourceHandlerImpl.java | 1 + .../TestCGroupsMemoryResourceHandlerImpl.java | 7 +- .../linux/resources/TestDefaultOOMHandler.java | 307 +++++++ .../runtime/TestDockerContainerRuntime.java | 470 ++++++----- .../linux/runtime/docker/TestDockerClient.java | 80 ++ .../TestAppLogAggregatorImpl.java | 75 +- .../monitor/TestContainersMonitor.java | 1 + .../TestContainersMonitorResourceChange.java | 3 +- .../TestResourcePluginManager.java | 6 + .../TestContainerSchedulerQueuing.java | 110 +-- .../TestNMTimelinePublisher.java | 102 ++- .../nodemanager/webapp/TestNMWebServices.java | 7 +- .../webapp/TestNMWebServicesApps.java | 68 +- .../ApplicationMasterService.java | 2 +- .../server/resourcemanager/ClientRMService.java | 14 +- ...pportunisticContainerAllocatorAMService.java | 12 + .../server/resourcemanager/RMAppManager.java | 5 - .../server/resourcemanager/ResourceManager.java | 1 + .../metrics/TimelineServiceV2Publisher.java | 4 + .../AbstractPreemptableResourceCalculator.java | 58 +- .../CapacitySchedulerPreemptionUtils.java | 61 +- .../capacity/FifoCandidatesSelector.java | 8 +- .../FifoIntraQueuePreemptionPlugin.java | 41 +- .../capacity/IntraQueueCandidatesSelector.java | 42 +- .../capacity/PreemptableResourceCalculator.java | 13 +- .../monitor/capacity/TempAppPerPartition.java | 9 + .../monitor/capacity/TempQueuePerPartition.java | 8 +- .../reservation/NoOverCommitPolicy.java | 5 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 5 + .../scheduler/AbstractYarnScheduler.java | 11 +- .../scheduler/ResourceScheduler.java | 5 + .../scheduler/SchedulerUtils.java | 186 ++++- .../scheduler/capacity/AbstractCSQueue.java | 15 + .../scheduler/capacity/CapacityScheduler.java | 38 +- .../capacity/CapacitySchedulerMetrics.java | 119 +++ .../scheduler/capacity/UsersManager.java | 9 +- .../fair/AllocationFileLoaderService.java | 16 +- .../scheduler/fair/FSAppAttempt.java | 10 +- .../scheduler/fair/FSLeafQueue.java | 31 + .../scheduler/fair/FSParentQueue.java | 3 + .../resourcemanager/scheduler/fair/FSQueue.java | 9 + .../scheduler/fair/FairScheduler.java | 144 +++- .../scheduler/fair/QueueManager.java | 155 +++- .../allocation/AllocationFileQueueParser.java | 5 +- .../SingleConstraintAppPlacementAllocator.java | 187 +++-- .../AbstractComparatorOrderingPolicy.java | 2 - .../resourcemanager/webapp/RMWebServices.java | 310 +++---- .../server/resourcemanager/TestAppManager.java | 4 - .../TestApplicationMasterLauncher.java | 2 +- .../TestApplicationMasterService.java | 185 ++++- .../TestCapacitySchedulerMetrics.java | 110 +++ ...pportunisticContainerAllocatorAMService.java | 10 +- .../server/resourcemanager/TestRMRestart.java | 1 + .../TestSystemMetricsPublisherForV2.java | 3 + ...alCapacityPreemptionPolicyMockFramework.java | 42 +- .../TestPreemptionForQueueWithPriorities.java | 103 ++- ...pacityPreemptionPolicyInterQueueWithDRF.java | 114 ++- ...yPreemptionPolicyIntraQueueFairOrdering.java | 276 +++++++ .../scheduler/TestAbstractYarnScheduler.java | 60 +- .../scheduler/TestSchedulerUtils.java | 278 ++++++- .../scheduler/capacity/TestQueueState.java | 60 ++ ...estSchedulingRequestContainerAllocation.java | 84 ++ .../scheduler/fair/FairSchedulerTestBase.java | 64 +- .../fair/TestAllocationFileLoaderService.java | 110 +-- .../scheduler/fair/TestFairScheduler.java | 205 +++++ .../scheduler/fair/TestQueueManager.java | 337 ++++++++ .../webapp/TestRMWebServices.java | 2 +- ...tRMWebServicesHttpStaticUserPermissions.java | 12 +- .../storage/TestHBaseTimelineStorageApps.java | 4 +- .../storage/TestHBaseTimelineStorageDomain.java | 134 +++ .../TestHBaseTimelineStorageEntities.java | 4 +- .../storage/TestHBaseTimelineStorageSchema.java | 8 + .../flow/TestHBaseStorageFlowActivity.java | 4 +- .../storage/flow/TestHBaseStorageFlowRun.java | 4 +- .../flow/TestHBaseStorageFlowRunCompaction.java | 4 +- .../storage/HBaseTimelineWriterImpl.java | 46 ++ .../storage/domain/DomainTableRW.java | 2 +- .../collector/TimelineCollector.java | 28 + .../collector/TimelineCollectorWebService.java | 50 ++ .../reader/TimelineReaderServer.java | 9 +- .../reader/TimelineReaderWebServices.java | 33 + ...elineReaderWhitelistAuthorizationFilter.java | 14 +- .../storage/FileSystemTimelineWriterImpl.java | 9 + .../timelineservice/storage/TimelineWriter.java | 15 + .../collector/TestTimelineCollector.java | 32 + ...elineReaderWhitelistAuthorizationFilter.java | 58 +- .../hadoop-yarn-server-web-proxy/pom.xml | 13 + .../server/webproxy/amfilter/AmIpFilter.java | 19 +- .../webproxy/amfilter/TestSecureAmFilter.java | 159 ++++ .../src/test/resources/krb5.conf | 33 + .../src/site/markdown/DockerContainers.md | 3 +- .../site/markdown/NodeManagerCGroupsMemory.md | 133 +++ .../src/site/markdown/ResourceManagerRest.md | 118 ++- .../src/site/markdown/ResourceModel.md | 24 +- .../markdown/yarn-service/ServiceDiscovery.md | 14 +- .../markdown/yarn-service/YarnServiceAPI.md | 5 +- .../src/main/webapp/app/adapters/yarn-app.js | 13 +- .../main/webapp/app/adapters/yarn-servicedef.js | 4 +- .../webapp/app/components/deploy-service.js | 5 + .../main/webapp/app/components/timeline-view.js | 6 +- .../webapp/app/controllers/yarn-app-attempt.js | 2 +- .../src/main/webapp/app/controllers/yarn-app.js | 37 +- .../app/controllers/yarn-component-instance.js | 2 +- .../main/webapp/app/controllers/yarn-tools.js | 2 +- .../src/main/webapp/app/helpers/node-menu.js | 4 +- .../src/main/webapp/app/initializers/loader.js | 86 +- .../src/main/webapp/app/models/yarn-app.js | 2 + .../main/webapp/app/models/yarn-container.js | 6 + .../main/webapp/app/models/yarn-servicedef.js | 4 - .../app/models/yarn-timeline-appattempt.js | 4 + .../app/models/yarn-timeline-container.js | 6 + .../webapp/app/routes/yarn-node-container.js | 1 + .../webapp/app/serializers/yarn-app-timeline.js | 4 +- .../src/main/webapp/app/serializers/yarn-app.js | 4 +- .../webapp/app/serializers/yarn-container.js | 1 + .../app/serializers/yarn-timeline-appattempt.js | 6 +- .../app/serializers/yarn-timeline-container.js | 3 +- .../src/main/webapp/app/services/hosts.js | 5 + .../webapp/app/templates/cluster-overview.hbs | 4 +- .../templates/components/container-table.hbs | 2 +- .../app/templates/components/deploy-service.hbs | 18 +- .../templates/components/node-menu-panel.hbs | 4 +- .../components/service-component-table.hbs | 6 - .../yarn-queue/capacity-queue-info.hbs | 2 +- .../components/yarn-queue/capacity-queue.hbs | 2 +- .../components/yarn-queue/fair-queue-info.hbs | 2 +- .../components/yarn-queue/fair-queue.hbs | 2 +- .../src/main/webapp/app/templates/yarn-app.hbs | 40 +- .../main/webapp/app/templates/yarn-node-app.hbs | 2 +- .../app/templates/yarn-node-container.hbs | 2 +- .../main/webapp/app/templates/yarn-tools.hbs | 4 +- .../main/webapp/app/utils/href-address-utils.js | 2 +- .../src/main/webapp/config/default-config.js | 3 +- pom.xml | 43 +- 880 files changed, 36155 insertions(+), 20005 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/64ed2aa4/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/64ed2aa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/64ed2aa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
