Merge remote-tracking branch 'origin/trunk' into HDFS-12943 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3426d652 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3426d652 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3426d652
Branch: refs/heads/HDFS-12943 Commit: 3426d65259358489089ffd07534977bc068911c4 Parents: 362f1eb 632aca5 Author: Konstantin V Shvachko <s...@apache.org> Authored: Wed Jul 11 15:33:38 2018 -0700 Committer: Konstantin V Shvachko <s...@apache.org> Committed: Wed Jul 11 15:33:38 2018 -0700 ---------------------------------------------------------------------- .gitignore | 4 + dev-support/bin/ozone-dist-layout-stitching | 2 +- .../hadoop-client-minicluster/pom.xml | 8 + ...ExcludePrivateAnnotationsStandardDoclet.java | 6 +- .../hadoop-common/src/main/conf/hadoop-env.sh | 12 +- .../src/main/conf/hadoop-metrics2.properties | 2 +- .../org/apache/hadoop/conf/Configuration.java | 458 +++--- .../crypto/key/kms/KMSClientProvider.java | 4 +- .../java/org/apache/hadoop/fs/FileContext.java | 9 +- .../java/org/apache/hadoop/security/Groups.java | 5 +- .../hadoop-common/src/site/markdown/Metrics.md | 39 +- .../org/apache/hadoop/fs/TestFileContext.java | 44 +- .../src/main/conf/kms-log4j.properties | 4 +- .../src/test/resources/log4j.properties | 4 +- .../src/main/compose/ozone/docker-compose.yaml | 6 +- .../src/main/compose/ozone/docker-config | 2 +- .../src/main/compose/ozoneperf/README.md | 4 +- .../main/compose/ozoneperf/docker-compose.yaml | 6 +- .../src/main/compose/ozoneperf/docker-config | 2 +- .../scm/client/ContainerOperationClient.java | 117 +- hadoop-hdds/common/pom.xml | 18 + .../hadoop/hdds/protocol/DatanodeDetails.java | 13 +- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 7 +- .../hadoop/hdds/scm/client/ScmClient.java | 43 +- .../container/common/helpers/ContainerInfo.java | 167 ++- .../common/helpers/ContainerWithPipeline.java | 131 ++ .../StorageContainerLocationProtocol.java | 18 +- ...rLocationProtocolClientSideTranslatorPB.java | 34 +- .../scm/storage/ContainerProtocolCalls.java | 24 +- .../apache/hadoop/ozone/OzoneConfigKeys.java | 12 + .../org/apache/hadoop/ozone/OzoneConsts.java | 40 +- .../apache/hadoop/ozone/audit/AuditAction.java | 30 + .../hadoop/ozone/audit/AuditEventStatus.java | 36 + .../apache/hadoop/ozone/audit/AuditLogger.java | 128 ++ .../hadoop/ozone/audit/AuditLoggerType.java | 37 + .../apache/hadoop/ozone/audit/AuditMarker.java | 38 + .../apache/hadoop/ozone/audit/Auditable.java | 32 + .../apache/hadoop/ozone/audit/package-info.java | 123 ++ .../org/apache/hadoop/ozone/common/Storage.java | 9 +- ...rLocationProtocolServerSideTranslatorPB.java | 33 +- .../hadoop/utils/MetadataStoreBuilder.java | 40 +- .../main/proto/DatanodeContainerProtocol.proto | 18 +- .../main/proto/ScmBlockLocationProtocol.proto | 10 +- .../StorageContainerLocationProtocol.proto | 34 +- hadoop-hdds/common/src/main/proto/hdds.proto | 28 +- .../common/src/main/resources/ozone-default.xml | 164 ++- .../apache/hadoop/ozone/TestMetadataStore.java | 484 ------- .../apache/hadoop/ozone/audit/DummyAction.java | 51 + .../apache/hadoop/ozone/audit/DummyEntity.java | 57 + .../ozone/audit/TestOzoneAuditLogger.java | 147 ++ .../apache/hadoop/ozone/audit/package-info.java | 23 + .../apache/hadoop/utils/TestMetadataStore.java | 533 +++++++ .../common/src/test/resources/log4j2.properties | 76 + hadoop-hdds/container-service/pom.xml | 6 + .../apache/hadoop/hdds/scm/HddsServerUtil.java | 11 - .../container/common/DataNodeLayoutVersion.java | 80 ++ .../container/common/helpers/ChunkUtils.java | 344 ----- .../container/common/helpers/ContainerData.java | 512 ------- .../common/helpers/ContainerMetrics.java | 2 +- .../common/helpers/ContainerUtils.java | 328 ++--- .../common/helpers/DatanodeVersionFile.java | 95 ++ .../container/common/helpers/FileUtils.java | 82 -- .../container/common/helpers/KeyUtils.java | 148 -- .../common/impl/ChunkLayOutVersion.java | 98 ++ .../container/common/impl/ChunkManagerImpl.java | 233 --- .../container/common/impl/ContainerData.java | 477 +++++++ .../common/impl/ContainerDataYaml.java | 277 ++++ .../impl/ContainerLocationManagerImpl.java | 158 -- .../common/impl/ContainerManagerImpl.java | 1120 --------------- .../container/common/impl/ContainerSet.java | 255 ++++ .../common/impl/ContainerStorageLocation.java | 212 --- .../ozone/container/common/impl/Dispatcher.java | 687 --------- .../container/common/impl/HddsDispatcher.java | 208 +++ .../container/common/impl/KeyManagerImpl.java | 204 --- .../RandomContainerDeletionChoosingPolicy.java | 1 - ...NOrderedContainerDeletionChoosingPolicy.java | 1 - .../common/interfaces/ChunkManager.java | 73 - .../container/common/interfaces/Container.java | 100 ++ .../ContainerDeletionChoosingPolicy.java | 3 +- .../common/interfaces/ContainerDispatcher.java | 14 + .../common/interfaces/ContainerManager.java | 269 ---- .../container/common/interfaces/Handler.java | 74 + .../container/common/interfaces/KeyManager.java | 73 - .../common/interfaces/VolumeChoosingPolicy.java | 46 + .../statemachine/DatanodeStateMachine.java | 4 +- .../background/BlockDeletingService.java | 246 ---- .../statemachine/background/package-info.java | 18 - .../DeleteBlocksCommandHandler.java | 61 +- .../states/datanode/RunningDatanodeState.java | 3 +- .../states/endpoint/VersionEndpointTask.java | 31 +- .../server/ratis/ContainerStateMachine.java | 5 +- .../container/common/utils/ContainerCache.java | 6 +- .../container/common/utils/HddsVolumeUtil.java | 163 +++ .../container/common/volume/HddsVolume.java | 342 +++++ .../volume/RoundRobinVolumeChoosingPolicy.java | 83 ++ .../container/common/volume/VolumeIOStats.java | 139 ++ .../container/common/volume/VolumeInfo.java | 132 ++ .../container/common/volume/VolumeSet.java | 357 +++++ .../container/common/volume/VolumeUsage.java | 198 +++ .../container/common/volume/package-info.java | 21 + .../container/keyvalue/KeyValueContainer.java | 483 +++++++ .../keyvalue/KeyValueContainerData.java | 280 ++++ .../container/keyvalue/KeyValueHandler.java | 694 +++++++++ .../container/keyvalue/helpers/ChunkUtils.java | 357 +++++ .../container/keyvalue/helpers/KeyUtils.java | 147 ++ .../helpers/KeyValueContainerLocationUtil.java | 140 ++ .../keyvalue/helpers/KeyValueContainerUtil.java | 316 ++++ .../keyvalue/helpers/SmallFileUtils.java | 84 ++ .../keyvalue/helpers/package-info.java | 21 + .../keyvalue/impl/ChunkManagerImpl.java | 254 ++++ .../container/keyvalue/impl/KeyManagerImpl.java | 198 +++ .../container/keyvalue/impl/package-info.java | 22 + .../keyvalue/interfaces/ChunkManager.java | 80 ++ .../keyvalue/interfaces/KeyManager.java | 76 + .../ozone/container/keyvalue/package-info.java | 21 + .../background/BlockDeletingService.java | 248 ++++ .../statemachine/background/package-info.java | 18 + .../container/ozoneimpl/ContainerReader.java | 177 +++ .../container/ozoneimpl/OzoneContainer.java | 396 +++-- .../protocol/StorageContainerNodeProtocol.java | 4 +- .../hadoop/ozone/protocol/VersionResponse.java | 4 + .../ozone/container/common/SCMTestUtils.java | 13 +- .../ozone/container/common/ScmTestMock.java | 4 + .../common/TestChunkLayOutVersion.java | 42 + .../common/TestDatanodeLayOutVersion.java | 38 + .../common/TestKeyValueContainerData.java | 85 ++ .../common/helpers/TestDatanodeVersionFile.java | 134 ++ .../common/impl/TestContainerDataYaml.java | 167 +++ .../container/common/impl/TestContainerSet.java | 179 +++ .../common/interfaces/TestHandler.java | 89 ++ .../container/common/volume/TestHddsVolume.java | 145 ++ .../TestRoundRobinVolumeChoosingPolicy.java | 127 ++ .../container/common/volume/TestVolumeSet.java | 153 ++ .../keyvalue/TestChunkManagerImpl.java | 290 ++++ .../container/keyvalue/TestKeyManagerImpl.java | 189 +++ .../keyvalue/TestKeyValueContainer.java | 267 ++++ .../container/keyvalue/TestKeyValueHandler.java | 201 +++ .../container/ozoneimpl/TestOzoneContainer.java | 108 ++ .../testutils/BlockDeletingServiceTestImpl.java | 9 +- .../test/resources/additionalfields.container | 11 + .../src/test/resources/incorrect.container | 10 + hadoop-hdds/framework/pom.xml | 5 + .../hadoop/hdds/server/events/EventQueue.java | 108 +- .../hadoop/hdds/server/events/EventWatcher.java | 43 +- .../hdds/server/events/EventWatcherMetrics.java | 79 + .../server/events/SingleThreadExecutor.java | 35 +- .../src/main/resources/webapps/static/ozone.js | 4 +- .../webapps/static/templates/config.html | 4 +- .../hdds/server/events/TestEventQueue.java | 35 +- .../hdds/server/events/TestEventWatcher.java | 107 +- hadoop-hdds/pom.xml | 2 + .../hadoop/hdds/scm/block/BlockManagerImpl.java | 80 +- .../block/DatanodeDeletedBlockTransactions.java | 11 +- .../hadoop/hdds/scm/block/DeletedBlockLog.java | 2 +- .../container/CloseContainerEventHandler.java | 42 +- .../hdds/scm/container/ContainerMapping.java | 133 +- .../scm/container/ContainerReportHandler.java | 47 + .../scm/container/ContainerStateManager.java | 30 +- .../hadoop/hdds/scm/container/Mapping.java | 25 +- .../scm/container/closer/ContainerCloser.java | 16 +- .../scm/container/states/ContainerStateMap.java | 13 +- .../hadoop/hdds/scm/events/SCMEvents.java | 102 ++ .../hadoop/hdds/scm/events/package-info.java | 23 + .../hadoop/hdds/scm/node/CommandQueue.java | 2 +- .../hadoop/hdds/scm/node/DatanodeInfo.java | 120 ++ .../hadoop/hdds/scm/node/DeadNodeHandler.java | 42 + .../hdds/scm/node/HeartbeatQueueItem.java | 98 -- .../hadoop/hdds/scm/node/NewNodeHandler.java | 50 + .../hadoop/hdds/scm/node/NodeManager.java | 18 +- .../hadoop/hdds/scm/node/NodeReportHandler.java | 42 + .../hadoop/hdds/scm/node/NodeStateManager.java | 603 ++++++++ .../hadoop/hdds/scm/node/SCMNodeManager.java | 535 +------ .../hadoop/hdds/scm/node/StaleNodeHandler.java | 42 + .../node/states/NodeAlreadyExistsException.java | 45 + .../hdds/scm/node/states/NodeException.java | 44 + .../scm/node/states/NodeNotFoundException.java | 49 + .../hdds/scm/node/states/NodeStateMap.java | 281 ++++ .../hdds/scm/pipelines/PipelineManager.java | 27 +- .../hdds/scm/pipelines/PipelineSelector.java | 16 + .../scm/pipelines/ratis/RatisManagerImpl.java | 1 + .../standalone/StandaloneManagerImpl.java | 1 + .../hdds/scm/server/SCMBlockProtocolServer.java | 2 +- .../scm/server/SCMClientProtocolServer.java | 74 +- .../server/SCMDatanodeHeartbeatDispatcher.java | 33 +- .../scm/server/SCMDatanodeProtocolServer.java | 18 +- .../scm/server/StorageContainerManager.java | 50 +- .../hdds/scm/block/TestDeletedBlockLog.java | 15 +- .../hdds/scm/container/MockNodeManager.java | 67 +- .../TestCloseContainerEventHandler.java | 56 +- .../scm/container/TestContainerMapping.java | 27 +- .../container/closer/TestContainerCloser.java | 18 +- .../hdds/scm/node/TestContainerPlacement.java | 28 +- .../hadoop/hdds/scm/node/TestNodeManager.java | 197 +-- .../TestSCMDatanodeHeartbeatDispatcher.java | 28 +- .../ozone/container/common/TestEndPoint.java | 20 +- .../testutils/ReplicationNodeManagerMock.java | 44 +- .../hadoop/hdds/scm/cli/OzoneBaseCLI.java | 2 +- .../cli/container/CloseContainerHandler.java | 10 +- .../cli/container/DeleteContainerHandler.java | 9 +- .../scm/cli/container/InfoContainerHandler.java | 11 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 19 - .../org/apache/hadoop/hdfs/DFSInputStream.java | 46 +- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 15 +- .../hdfs/client/HdfsClientConfigKeys.java | 3 + .../hdfs/client/impl/BlockReaderFactory.java | 21 +- .../hdfs/client/impl/BlockReaderLocal.java | 93 +- .../client/impl/BlockReaderLocalLegacy.java | 44 +- .../hdfs/client/impl/BlockReaderRemote.java | 33 +- .../datanode/ReplicaNotFoundException.java | 2 +- .../ha/ConfiguredFailoverProxyProvider.java | 9 +- .../InMemoryAliasMapFailoverProxyProvider.java | 38 + .../main/native/libhdfspp/lib/rpc/request.cc | 2 +- .../hdfs/server/federation/router/Quota.java | 10 +- .../router/RouterQuotaUpdateService.java | 43 +- .../federation/router/RouterRpcServer.java | 1 - .../src/site/markdown/HDFSRouterFederation.md | 4 +- .../router/TestDisableRouterQuota.java | 94 ++ .../federation/router/TestRouterQuota.java | 212 ++- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 37 +- .../org/apache/hadoop/hdfs/NameNodeProxies.java | 15 +- ...yAliasMapProtocolClientSideTranslatorPB.java | 95 +- .../aliasmap/InMemoryAliasMapProtocol.java | 5 + .../aliasmap/InMemoryLevelDBAliasMapServer.java | 19 +- .../impl/InMemoryLevelDBAliasMapClient.java | 80 +- .../hadoop/hdfs/server/datanode/DataNode.java | 21 +- .../hdfs/server/datanode/DiskBalancer.java | 29 +- .../erasurecode/StripedBlockReader.java | 2 +- .../fsdataset/VolumeChoosingPolicy.java | 2 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 8 + .../hdfs/server/namenode/NamenodeFsck.java | 1 - .../src/main/resources/hdfs-default.xml | 35 +- .../src/site/markdown/HDFSCommands.md | 4 +- .../src/site/markdown/HdfsProvidedStorage.md | 2 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 13 +- .../apache/hadoop/hdfs/MiniDFSNNTopology.java | 2 +- .../hdfs/client/impl/BlockReaderTestUtil.java | 2 - .../hdfs/client/impl/TestBlockReaderLocal.java | 2 - .../blockmanagement/TestBlockTokenWithDFS.java | 2 - .../TestNameNodePrunesMissingStorages.java | 5 +- .../impl/TestInMemoryLevelDBAliasMapClient.java | 7 + .../datanode/TestDataNodeVolumeFailure.java | 2 - .../server/diskbalancer/TestDiskBalancer.java | 80 +- .../shortcircuit/TestShortCircuitCache.java | 89 ++ .../security/TestRefreshUserMappings.java | 19 +- .../src/test/acceptance/basic/basic.robot | 6 +- .../test/acceptance/basic/docker-compose.yaml | 8 +- .../src/test/acceptance/basic/docker-config | 4 +- .../src/test/acceptance/basic/ozone-shell.robot | 18 +- .../src/test/acceptance/commonlib.robot | 4 +- .../test/acceptance/ozonefs/docker-compose.yaml | 8 +- .../src/test/acceptance/ozonefs/docker-config | 4 +- .../src/test/acceptance/ozonefs/ozonefs.robot | 6 +- .../apache/hadoop/ozone/client/BucketArgs.java | 4 +- .../hadoop/ozone/client/OzoneClientFactory.java | 89 +- .../apache/hadoop/ozone/client/OzoneKey.java | 2 +- .../apache/hadoop/ozone/client/VolumeArgs.java | 4 +- .../ozone/client/io/ChunkGroupInputStream.java | 39 +- .../ozone/client/io/ChunkGroupOutputStream.java | 63 +- .../client/io/OzoneContainerTranslation.java | 50 - .../client/rest/DefaultRestServerSelector.java | 2 +- .../hadoop/ozone/client/rest/RestClient.java | 15 +- .../ozone/client/rest/RestServerSelector.java | 2 +- .../hadoop/ozone/client/rpc/RpcClient.java | 142 +- .../ozone/client/TestHddsClientUtils.java | 24 +- hadoop-ozone/common/pom.xml | 2 +- hadoop-ozone/common/src/main/bin/ozone | 9 +- hadoop-ozone/common/src/main/bin/start-ozone.sh | 16 +- hadoop-ozone/common/src/main/bin/stop-ozone.sh | 16 +- .../java/org/apache/hadoop/ozone/KsmUtils.java | 87 -- .../java/org/apache/hadoop/ozone/OmUtils.java | 94 ++ .../org/apache/hadoop/ozone/audit/OMAction.java | 51 + .../apache/hadoop/ozone/audit/package-info.java | 22 + .../apache/hadoop/ozone/freon/OzoneGetConf.java | 16 +- .../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 81 -- .../hadoop/ozone/ksm/helpers/KsmBucketArgs.java | 233 --- .../hadoop/ozone/ksm/helpers/KsmBucketInfo.java | 235 --- .../hadoop/ozone/ksm/helpers/KsmKeyArgs.java | 119 -- .../hadoop/ozone/ksm/helpers/KsmKeyInfo.java | 277 ---- .../ozone/ksm/helpers/KsmKeyLocationInfo.java | 129 -- .../ksm/helpers/KsmKeyLocationInfoGroup.java | 118 -- .../ozone/ksm/helpers/KsmOzoneAclMap.java | 110 -- .../hadoop/ozone/ksm/helpers/KsmVolumeArgs.java | 223 --- .../ozone/ksm/helpers/OpenKeySession.java | 50 - .../hadoop/ozone/ksm/helpers/ServiceInfo.java | 237 --- .../hadoop/ozone/ksm/helpers/VolumeArgs.java | 140 -- .../hadoop/ozone/ksm/helpers/package-info.java | 18 - .../apache/hadoop/ozone/ksm/package-info.java | 21 - .../ksm/protocol/KeySpaceManagerProtocol.java | 252 ---- .../hadoop/ozone/ksm/protocol/package-info.java | 19 - ...ceManagerProtocolClientSideTranslatorPB.java | 769 ---------- .../protocolPB/KeySpaceManagerProtocolPB.java | 34 - .../ozone/ksm/protocolPB/package-info.java | 19 - .../apache/hadoop/ozone/om/OMConfigKeys.java | 81 ++ .../hadoop/ozone/om/helpers/OmBucketArgs.java | 233 +++ .../hadoop/ozone/om/helpers/OmBucketInfo.java | 235 +++ .../hadoop/ozone/om/helpers/OmKeyArgs.java | 119 ++ .../hadoop/ozone/om/helpers/OmKeyInfo.java | 277 ++++ .../ozone/om/helpers/OmKeyLocationInfo.java | 129 ++ .../om/helpers/OmKeyLocationInfoGroup.java | 118 ++ .../hadoop/ozone/om/helpers/OmOzoneAclMap.java | 110 ++ .../hadoop/ozone/om/helpers/OmVolumeArgs.java | 223 +++ .../hadoop/ozone/om/helpers/OpenKeySession.java | 50 + .../hadoop/ozone/om/helpers/ServiceInfo.java | 237 +++ .../hadoop/ozone/om/helpers/VolumeArgs.java | 140 ++ .../hadoop/ozone/om/helpers/package-info.java | 18 + .../apache/hadoop/ozone/om/package-info.java | 21 + .../ozone/om/protocol/OzoneManagerProtocol.java | 252 ++++ .../hadoop/ozone/om/protocol/package-info.java | 19 + ...neManagerProtocolClientSideTranslatorPB.java | 769 ++++++++++ .../om/protocolPB/OzoneManagerProtocolPB.java | 34 + .../ozone/om/protocolPB/package-info.java | 19 + .../hadoop/ozone/protocolPB/KSMPBHelper.java | 113 -- .../hadoop/ozone/protocolPB/OMPBHelper.java | 113 ++ .../hadoop/ozone/protocolPB/OzonePBHelper.java | 30 + .../main/proto/KeySpaceManagerProtocol.proto | 474 ------ .../src/main/proto/OzoneManagerProtocol.proto | 480 +++++++ hadoop-ozone/docs/content/GettingStarted.md | 18 +- hadoop-ozone/docs/content/Metrics.md | 10 +- hadoop-ozone/docs/content/_index.md | 12 +- hadoop-ozone/docs/static/OzoneOverview.svg | 2 +- .../container/TestContainerStateManager.java | 161 ++- .../apache/hadoop/ozone/MiniOzoneCluster.java | 24 +- .../hadoop/ozone/MiniOzoneClusterImpl.java | 66 +- .../hadoop/ozone/TestContainerOperations.java | 11 +- .../ozone/TestOzoneConfigurationFields.java | 4 +- .../ozone/TestStorageContainerManager.java | 28 +- .../TestStorageContainerManagerHelper.java | 33 +- .../ozone/client/rest/TestOzoneRestClient.java | 6 +- .../ozone/client/rpc/TestOzoneRpcClient.java | 22 +- .../ozone/container/ContainerTestHelper.java | 27 +- .../common/TestBlockDeletingService.java | 118 +- .../TestContainerDeletionChoosingPolicy.java | 63 +- .../common/impl/TestContainerPersistence.java | 478 +++---- .../commandhandler/TestBlockDeletion.java | 77 +- .../TestCloseContainerByPipeline.java | 62 +- .../TestCloseContainerHandler.java | 20 +- .../container/metrics/TestContainerMetrics.java | 60 +- .../container/ozoneimpl/TestOzoneContainer.java | 19 +- .../container/server/TestContainerServer.java | 24 +- ...TestGenerateOzoneRequiredConfigurations.java | 5 +- .../ozone/ksm/TestContainerReportWithKeys.java | 143 -- .../apache/hadoop/ozone/ksm/TestKSMMetrcis.java | 306 ---- .../apache/hadoop/ozone/ksm/TestKSMSQLCli.java | 284 ---- .../hadoop/ozone/ksm/TestKeySpaceManager.java | 1350 ------------------ .../ksm/TestKeySpaceManagerRestInterface.java | 135 -- .../ozone/ksm/TestKsmBlockVersioning.java | 253 ---- .../ksm/TestMultipleContainerReadWrite.java | 215 --- .../ozone/om/TestContainerReportWithKeys.java | 143 ++ .../om/TestMultipleContainerReadWrite.java | 215 +++ .../hadoop/ozone/om/TestOmBlockVersioning.java | 253 ++++ .../apache/hadoop/ozone/om/TestOmMetrics.java | 313 ++++ .../apache/hadoop/ozone/om/TestOmSQLCli.java | 284 ++++ .../hadoop/ozone/om/TestOzoneManager.java | 1349 +++++++++++++++++ .../ozone/om/TestOzoneManagerRestInterface.java | 135 ++ .../hadoop/ozone/ozShell/TestOzoneShell.java | 40 +- .../hadoop/ozone/scm/TestAllocateContainer.java | 6 +- .../hadoop/ozone/scm/TestContainerSQLCli.java | 3 +- .../ozone/scm/TestContainerSmallFile.java | 38 +- .../org/apache/hadoop/ozone/scm/TestSCMCli.java | 161 ++- .../ozone/scm/TestXceiverClientManager.java | 62 +- .../ozone/scm/TestXceiverClientMetrics.java | 14 +- .../hadoop/ozone/scm/node/TestQueryNode.java | 19 +- .../ozone/web/TestDistributedOzoneVolumes.java | 12 +- .../hadoop/ozone/web/client/TestKeys.java | 82 +- .../src/test/resources/webapps/ksm/.gitkeep | 15 - .../resources/webapps/ozoneManager/.gitkeep | 15 + .../server/datanode/ObjectStoreHandler.java | 33 +- .../ozone/web/handlers/KeyProcessTemplate.java | 4 +- .../web/handlers/VolumeProcessTemplate.java | 4 +- .../web/storage/DistributedStorageHandler.java | 153 +- .../apache/hadoop/ozone/ksm/BucketManager.java | 79 - .../hadoop/ozone/ksm/BucketManagerImpl.java | 315 ---- .../org/apache/hadoop/ozone/ksm/KSMMXBean.java | 31 - .../hadoop/ozone/ksm/KSMMetadataManager.java | 253 ---- .../ozone/ksm/KSMMetadataManagerImpl.java | 526 ------- .../org/apache/hadoop/ozone/ksm/KSMMetrics.java | 459 ------ .../org/apache/hadoop/ozone/ksm/KSMStorage.java | 90 -- .../hadoop/ozone/ksm/KeyDeletingService.java | 142 -- .../org/apache/hadoop/ozone/ksm/KeyManager.java | 175 --- .../apache/hadoop/ozone/ksm/KeyManagerImpl.java | 566 -------- .../hadoop/ozone/ksm/KeySpaceManager.java | 914 ------------ .../ozone/ksm/KeySpaceManagerHttpServer.java | 78 - .../hadoop/ozone/ksm/OpenKeyCleanupService.java | 117 -- .../ozone/ksm/ServiceListJSONServlet.java | 103 -- .../apache/hadoop/ozone/ksm/VolumeManager.java | 100 -- .../hadoop/ozone/ksm/VolumeManagerImpl.java | 391 ----- .../ozone/ksm/exceptions/KSMException.java | 118 -- .../ozone/ksm/exceptions/package-info.java | 19 - .../apache/hadoop/ozone/ksm/package-info.java | 21 - .../apache/hadoop/ozone/om/BucketManager.java | 79 + .../hadoop/ozone/om/BucketManagerImpl.java | 315 ++++ .../hadoop/ozone/om/KeyDeletingService.java | 142 ++ .../org/apache/hadoop/ozone/om/KeyManager.java | 175 +++ .../apache/hadoop/ozone/om/KeyManagerImpl.java | 566 ++++++++ .../org/apache/hadoop/ozone/om/OMMXBean.java | 31 + .../hadoop/ozone/om/OMMetadataManager.java | 253 ++++ .../org/apache/hadoop/ozone/om/OMMetrics.java | 459 ++++++ .../org/apache/hadoop/ozone/om/OMStorage.java | 90 ++ .../hadoop/ozone/om/OmMetadataManagerImpl.java | 526 +++++++ .../hadoop/ozone/om/OpenKeyCleanupService.java | 117 ++ .../apache/hadoop/ozone/om/OzoneManager.java | 911 ++++++++++++ .../hadoop/ozone/om/OzoneManagerHttpServer.java | 78 + .../hadoop/ozone/om/ServiceListJSONServlet.java | 103 ++ .../apache/hadoop/ozone/om/VolumeManager.java | 100 ++ .../hadoop/ozone/om/VolumeManagerImpl.java | 390 +++++ .../hadoop/ozone/om/exceptions/OMException.java | 118 ++ .../ozone/om/exceptions/package-info.java | 19 + .../apache/hadoop/ozone/om/package-info.java | 21 + ...ceManagerProtocolServerSideTranslatorPB.java | 559 -------- ...neManagerProtocolServerSideTranslatorPB.java | 571 ++++++++ .../hadoop/ozone/protocolPB/package-info.java | 2 +- .../web/ozShell/volume/CreateVolumeHandler.java | 10 +- .../src/main/webapps/ksm/index.html | 70 - .../src/main/webapps/ksm/ksm-metrics.html | 44 - .../ozone-manager/src/main/webapps/ksm/ksm.js | 110 -- .../ozone-manager/src/main/webapps/ksm/main.css | 23 - .../src/main/webapps/ksm/main.html | 18 - .../src/main/webapps/ozoneManager/index.html | 70 + .../src/main/webapps/ozoneManager/main.css | 23 + .../src/main/webapps/ozoneManager/main.html | 18 + .../main/webapps/ozoneManager/om-metrics.html | 44 + .../main/webapps/ozoneManager/ozoneManager.js | 110 ++ .../hadoop/ozone/ksm/TestBucketManagerImpl.java | 395 ----- .../hadoop/ozone/ksm/TestChunkStreams.java | 234 --- .../ksm/TestKeySpaceManagerHttpServer.java | 141 -- .../apache/hadoop/ozone/ksm/package-info.java | 21 - .../hadoop/ozone/om/TestBucketManagerImpl.java | 394 +++++ .../hadoop/ozone/om/TestChunkStreams.java | 234 +++ .../ozone/om/TestOzoneManagerHttpServer.java | 141 ++ .../apache/hadoop/ozone/om/package-info.java | 21 + .../hadoop/fs/ozone/contract/OzoneContract.java | 4 +- .../genesis/BenchMarkContainerStateMap.java | 16 +- .../genesis/BenchMarkDatanodeDispatcher.java | 41 +- .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 111 +- hadoop-project/src/site/markdown/index.md.vm | 2 +- .../apache/hadoop/fs/s3a/S3AInputStream.java | 24 +- .../hadoop/fs/s3a/s3guard/S3GuardTool.java | 10 + .../s3guard/AbstractS3GuardToolTestBase.java | 18 + .../org/apache/hadoop/tools/DistCpOptions.java | 5 +- .../hadoop/tools/mapred/CopyCommitter.java | 13 +- .../contract/AbstractContractDistCpTest.java | 2 +- .../namenode/ITestProvidedImplementation.java | 371 ++++- .../dev-support/findbugs-exclude.xml | 17 +- .../hadoop/yarn/api/records/Resource.java | 13 + .../api/records/impl/LightWeightResource.java | 23 +- .../hadoop/yarn/conf/YarnConfiguration.java | 7 + .../yarn/service/client/TestServiceCLI.java | 6 +- .../impl/pb/GetApplicationsRequestPBImpl.java | 44 +- .../hadoop/yarn/client/AMRMClientUtils.java | 91 -- .../logaggregation/AggregatedLogFormat.java | 6 +- .../timeline/RollingLevelDBTimelineStore.java | 6 + .../server/timeline/TimelineDataManager.java | 7 +- .../timeline/webapp/TimelineWebServices.java | 4 + .../webapp/TestTimelineWebServices.java | 2 +- .../hadoop/yarn/server/AMRMClientRelayer.java | 9 +- .../yarn/server/uam/UnmanagedAMPoolManager.java | 16 + .../server/uam/UnmanagedApplicationManager.java | 40 +- .../yarn/server/MockResourceManagerFacade.java | 13 +- .../amrmproxy/FederationInterceptor.java | 146 +- .../containermanager/ContainerManagerImpl.java | 69 +- .../application/ApplicationImpl.java | 43 +- .../runtime/DockerLinuxContainerRuntime.java | 4 +- .../runtime/ContainerExecutionException.java | 6 + .../impl/container-executor.c | 30 +- .../container-executor/impl/utils/docker-util.c | 2 +- .../test/test-container-executor.c | 20 + .../amrmproxy/BaseAMRMProxyTest.java | 2 + .../amrmproxy/TestFederationInterceptor.java | 17 + .../BaseContainerManagerTest.java | 25 + .../TestContainerManagerRecovery.java | 106 +- .../application/TestApplication.java | 53 +- .../runtime/TestDockerContainerRuntime.java | 10 +- .../conf/capacity-scheduler.xml | 10 + .../scheduler/capacity/CapacityScheduler.java | 45 +- .../CapacitySchedulerConfiguration.java | 10 + .../scheduler/capacity/ParentQueue.java | 36 +- .../allocator/AbstractContainerAllocator.java | 13 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 5 + .../scheduler/fair/ConfigurableResource.java | 69 +- .../fair/FairSchedulerConfiguration.java | 174 ++- .../allocation/AllocationFileQueueParser.java | 2 +- .../resourcemanager/webapp/RMWSConsts.java | 3 + .../resourcemanager/webapp/RMWebServices.java | 6 +- .../resourcemanager/webapp/dao/AppInfo.java | 2 +- .../webapp/dao/SchedulerInfo.java | 8 +- .../TestWorkPreservingRMRestart.java | 2 + .../fair/TestFairSchedulerConfiguration.java | 160 ++- .../webapp/TestRMWebServices.java | 31 +- .../webapp/TestRMWebServicesApps.java | 14 +- ...estRMWebServicesAppsCustomResourceTypes.java | 242 ++++ .../webapp/TestRMWebServicesCapacitySched.java | 30 +- .../TestRMWebServicesConfigurationMutation.java | 5 + .../webapp/TestRMWebServicesFairScheduler.java | 95 +- .../TestRMWebServicesSchedulerActivities.java | 2 +- ...ustomResourceTypesConfigurationProvider.java | 138 ++ .../FairSchedulerJsonVerifications.java | 139 ++ .../FairSchedulerXmlVerifications.java | 153 ++ ...ervicesFairSchedulerCustomResourceTypes.java | 271 ++++ .../webapp/helper/AppInfoJsonVerifications.java | 123 ++ .../webapp/helper/AppInfoXmlVerifications.java | 132 ++ .../webapp/helper/BufferedClientResponse.java | 57 + .../helper/JsonCustomResourceTypeTestcase.java | 77 + .../ResourceRequestsJsonVerifications.java | 252 ++++ .../ResourceRequestsXmlVerifications.java | 215 +++ .../helper/XmlCustomResourceTypeTestCase.java | 112 ++ .../router/clientrm/RouterClientRMService.java | 53 +- .../router/rmadmin/RouterRMAdminService.java | 51 +- .../server/router/webapp/RouterWebServices.java | 48 +- .../clientrm/TestRouterClientRMService.java | 60 + .../rmadmin/TestRouterRMAdminService.java | 60 + .../router/webapp/TestRouterWebServices.java | 65 + .../pom.xml | 10 + .../storage/TestTimelineReaderHBaseDown.java | 220 +++ .../storage/HBaseTimelineReaderImpl.java | 93 ++ .../src/site/markdown/FairScheduler.md | 6 +- 516 files changed, 33287 insertions(+), 21883 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3426d652/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3426d652/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3426d652/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
