This is an automated email from the ASF dual-hosted git repository. adoroszlai pushed a commit to branch HDDS-4440-s3-performance in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 3b4e2f58b35f1e2dbf553ea183063867f1fd4771 Merge: cf8eb33 db574d1 Author: Doroszlai, Attila <[email protected]> AuthorDate: Fri Aug 20 07:36:19 2021 +0200 Merge remote-tracking branch 'origin/master' into HDDS-4440-s3-performance .github/workflows/post-commit.yml | 130 ++- SECURITY.md | 1 + dev-support/ci/lib/_all_libs.sh | 34 + dev-support/ci/lib/_initialization.sh | 140 +++ dev-support/ci/lib/_script_init.sh | 42 + dev-support/ci/lib/_start_end.sh | 112 +++ dev-support/ci/lib/_traps.sh | 38 + dev-support/ci/lib/_verbosity.sh | 57 ++ dev-support/ci/selective_ci_checks.bats | 210 ++++ dev-support/ci/selective_ci_checks.sh | 522 ++++++++++ .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 18 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 72 +- .../hdds/scm/storage/RatisBlockOutputStream.java | 126 +++ .../storage/TestBlockOutputStreamCorrectness.java | 2 +- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 2 +- .../java/org/apache/hadoop/hdds/HddsUtils.java | 19 + .../hadoop/hdds/client/ReplicationConfig.java | 10 +- .../hdds/client/ReplicationConfigValidator.java | 63 ++ .../hadoop/hdds/protocol/DatanodeDetails.java | 9 + .../apache/hadoop/hdds/scm/PlacementPolicy.java | 7 +- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 80 ++ .../apache/hadoop/hdds/scm/client/ScmClient.java | 4 +- .../hadoop/hdds/scm/ha/SCMHAConfiguration.java | 320 ------ .../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 6 +- .../org/apache/hadoop/hdds/scm/net/InnerNode.java | 7 + .../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 27 + .../hadoop/hdds/scm/net/NetworkTopology.java | 7 + .../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 17 + .../protocol/StorageContainerLocationProtocol.java | 11 +- .../apache/hadoop/hdds/utils/HddsVersionInfo.java | 5 +- .../org/apache/hadoop/hdds/utils/VersionInfo.java | 10 +- .../apache/hadoop/ozone/common/MonotonicClock.java | 81 ++ .../hadoop/ozone/conf/OzoneServiceConfig.java | 76 ++ .../org/apache/hadoop/ozone/conf/package-info.java | 15 +- .../ozone/upgrade/BasicUpgradeFinalizer.java | 18 +- .../hadoop/ozone/util/ShutdownHookManager.java | 396 ++++++++ .../org/apache/hadoop/ozone/util/package-info.java | 15 +- .../main/resources/hdds-version-info.properties | 3 +- .../common/src/main/resources/ozone-default.xml | 160 ++- .../hadoop/hdds/client/TestReplicationConfig.java | 24 + .../client/TestReplicationConfigValidator.java | 73 ++ .../hdds/scm/net/TestNetworkTopologyImpl.java | 26 +- .../ozone/container/ContainerTestHelper.java | 75 ++ .../org/apache/hadoop/hdds/conf/StorageUnit.java | 8 +- .../apache/hadoop/hdds/conf/TimeDurationUtil.java | 3 + .../hadoop/hdds/freon/FakeRatisFollower.java | 7 +- .../apache/hadoop/ozone/HddsDatanodeService.java | 13 +- .../container/common/helpers/ContainerUtils.java | 5 +- .../container/common/report/ReportManager.java | 3 +- .../common/statemachine/DatanodeConfiguration.java | 52 +- .../common/statemachine/DatanodeStateMachine.java | 15 +- .../commandhandler/DeleteBlocksCommandHandler.java | 276 ++++-- .../common/transport/server/XceiverServerGrpc.java | 3 +- .../server/ratis/ContainerStateMachine.java | 8 +- .../transport/server/ratis/XceiverServerRatis.java | 20 +- .../common/utils/db/DatanodeDBProfile.java | 145 +++ .../container/common/utils/db/package-info.java | 17 +- .../common/volume/ImmutableVolumeSet.java | 16 + .../container/common/volume/MutableVolumeSet.java | 25 +- .../common/volume/StorageVolumeChecker.java | 54 +- .../ozone/container/common/volume/VolumeSet.java | 3 + .../container/metadata/AbstractDatanodeStore.java | 47 +- .../ozone/container/ozoneimpl/ContainerReader.java | 11 +- .../replication/GrpcReplicationClient.java | 3 +- .../OnDemandContainerReplicationSource.java | 10 +- .../container/replication/ReplicationServer.java | 1 + .../replication/ReplicationSupervisor.java | 33 +- .../container/stream/DirectoryServerSource.java | 12 +- .../ozone/container/stream/StreamingClient.java | 1 + .../ozone/container/stream/StreamingServer.java | 1 + .../upgrade/DataNodeUpgradeFinalizer.java | 4 +- .../common/helpers/TestContainerUtils.java | 60 ++ .../common/volume/TestPeriodicVolumeChecker.java | 131 +++ .../container/keyvalue/TestKeyValueContainer.java | 77 +- .../TestKeyValueHandlerWithUnhealthyContainer.java | 78 +- .../replication/TestReplicationSupervisor.java | 4 +- .../docs/content/feature/Nonrolling-Upgrade.md | 107 ++ hadoop-hdds/docs/content/feature/OM-HA.zh.md | 4 +- hadoop-hdds/docs/content/feature/Quota.zh.md | 4 +- hadoop-hdds/docs/content/feature/Recon.zh.md | 2 +- hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 4 +- hadoop-hdds/docs/content/feature/_index.zh.md | 14 +- .../feature/how-to-do-a-nonrolling-upgrade.md | 54 -- hadoop-hdds/docs/content/interface/Ofs.md | 28 +- hadoop-hdds/docs/content/security/SecuringTDE.md | 54 ++ .../java/org/apache/hadoop/hdds/NodeDetails.java | 44 +- .../container/common/helpers/MoveDataNodePair.java | 71 ++ .../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 6 + ...inerLocationProtocolClientSideTranslatorPB.java | 30 +- .../scm/update/client/CRLClientUpdateHandler.java | 5 +- .../update/client/SCMUpdateServiceGrpcClient.java | 3 +- .../org/apache/hadoop/hdds/server/JsonUtils.java | 2 + .../hadoop/hdds/server/http/HttpServer2.java | 18 +- .../hadoop/hdds/utils/MetadataKeyFilters.java | 2 +- .../org/apache/hadoop/hdds/utils/db/DBProfile.java | 35 +- .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 14 + .../hadoop/hdds/server/http/TestHttpServer2.java | 54 ++ .../apache/hadoop/hdds/utils/db/TestRDBStore.java | 2 +- hadoop-hdds/interface-admin/pom.xml | 2 +- .../src/main/proto/ScmAdminProtocol.proto | 4 +- .../dev-support/findbugsExcludeFile.xml | 3 + hadoop-hdds/interface-client/pom.xml | 38 +- .../interface-client/src/main/proto/hdds.proto | 5 + hadoop-hdds/interface-server/pom.xml | 2 +- .../src/main/proto/SCMRatisProtocol.proto | 1 + .../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 122 ++- .../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 10 + .../hadoop/hdds/scm/block/BlockManagerImpl.java | 27 +- .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 281 +++--- .../hdds/scm/block/DeletedBlockLogImplV2.java | 432 --------- .../hdds/scm/block/SCMBlockDeletingService.java | 18 +- .../scm/block/ScmBlockDeletingServiceMetrics.java | 186 ++++ .../scm/command/CommandStatusReportHandler.java | 6 +- .../hdds/scm/container/ContainerReplicaCount.java | 1 + .../hdds/scm/container/ContainerStateManager.java | 10 +- .../hdds/scm/container/ReplicationManager.java | 953 +++++++++++++++--- .../scm/container/balancer/ContainerBalancer.java | 637 +++++++++--- .../balancer/ContainerBalancerConfiguration.java | 206 +++- .../balancer/ContainerBalancerMetrics.java | 13 +- .../ContainerBalancerSelectionCriteria.java | 169 ++++ .../container/balancer/ContainerMoveSelection.java | 55 ++ .../scm/container/balancer/FindTargetGreedy.java | 135 +++ .../scm/container/balancer/FindTargetStrategy.java | 71 ++ .../algorithms/SCMContainerPlacementCapacity.java | 10 +- .../algorithms/SCMContainerPlacementRackAware.java | 86 +- .../algorithms/SCMContainerPlacementRandom.java | 10 +- .../replication/ReplicationManagerMetrics.java | 161 ++++ .../scm/container/states/ContainerQueryKey.java | 33 +- .../scm/container/states/ContainerStateMap.java | 41 +- .../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 1 + .../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 10 +- .../hdds/scm/ha/InterSCMGrpcProtocolService.java | 3 +- .../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 131 ++- .../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 6 +- .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 2 + .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 26 +- .../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 1 + .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 6 +- .../hdds/scm/metadata/MoveDataNodePairCodec.java | 53 + .../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 13 +- .../hdds/scm/metadata/SCMMetadataStoreImpl.java | 13 + .../hadoop/hdds/scm/node/DeadNodeHandler.java | 13 + .../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 12 + .../apache/hadoop/hdds/scm/node/NodeManager.java | 9 + .../apache/hadoop/hdds/scm/node/NodeStatus.java | 5 + .../hadoop/hdds/scm/node/SCMNodeManager.java | 27 + ...ManagerV2Impl.java => PipelineManagerImpl.java} | 22 +- .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 57 +- .../hadoop/hdds/scm/pipeline/PipelineProvider.java | 31 +- .../hdds/scm/pipeline/PipelineReportHandler.java | 2 + .../scm/pipeline/PipelineStateManagerV2Impl.java | 4 +- .../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 4 +- .../hdds/scm/pipeline/RatisPipelineProvider.java | 17 +- .../hdds/scm/pipeline/SCMPipelineManager.java | 801 --------------- .../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +- ...inerLocationProtocolServerSideTranslatorPB.java | 40 +- .../hdds/scm/server/SCMClientProtocolServer.java | 56 +- .../hdds/scm/server/SCMSecurityProtocolServer.java | 1 + .../hdds/scm/server/StorageContainerManager.java | 32 +- .../scm/server/StorageContainerManagerStarter.java | 12 +- .../scm/server/upgrade/SCMUpgradeFinalizer.java | 7 +- .../ScmHAUnfinalizedStateValidationAction.java | 13 +- .../update/server/SCMUpdateServiceGrpcServer.java | 3 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 6 +- .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 19 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 67 +- .../container/TestCloseContainerEventHandler.java | 6 +- .../hdds/scm/container/TestReplicationManager.java | 843 +++++++++++++--- .../scm/container/TestSCMContainerManager.java | 6 +- .../container/balancer/TestContainerBalancer.java | 466 ++++++++- .../algorithms/TestContainerPlacementFactory.java | 39 +- .../TestSCMContainerPlacementCapacity.java | 2 +- .../TestSCMContainerPlacementRackAware.java | 109 ++- .../TestSCMContainerPlacementRandom.java | 70 +- .../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 25 - .../hdds/scm/node/TestContainerPlacement.java | 4 +- .../hdds/scm/node/TestDatanodeAdminMonitor.java | 1 + .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 36 +- .../hdds/scm/node/states/TestNodeStateMap.java | 14 +- .../hdds/scm/pipeline/TestPipelineManagerImpl.java | 40 +- .../scm/pipeline/TestPipelinePlacementPolicy.java | 71 +- .../scm/pipeline/TestRatisPipelineProvider.java | 44 +- .../hdds/scm/pipeline/TestSCMPipelineManager.java | 821 ---------------- ...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 6 + .../safemode/TestHealthyPipelineSafeModeRule.java | 14 +- .../TestOneReplicaPipelineSafeModeRule.java | 6 +- .../hdds/scm/safemode/TestSCMSafeModeManager.java | 20 +- .../server/TestSCMUpdateServiceGrpcServer.java | 3 + .../ozone/container/common/TestEndPoint.java | 3 +- .../placement/TestContainerPlacement.java | 5 +- .../main/java/org/apache/ozone/test/TestClock.java | 76 ++ .../hdds/scm/cli/ContainerBalancerCommands.java | 19 +- .../scm/cli/ContainerBalancerStartSubcommand.java | 16 +- .../hdds/scm/cli/ContainerOperationClient.java | 26 +- .../hdds/scm/cli/SafeModeWaitSubcommand.java | 3 +- .../hdds/scm/cli/container/ContainerCommands.java | 1 - .../hdds/scm/cli/container/DeleteSubcommand.java | 54 -- .../hdds/scm/cli/datanode/ListInfoSubcommand.java | 22 + .../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 61 +- .../org/apache/hadoop/ozone/client/OzoneKey.java | 4 + .../ozone/client/io/BlockOutputStreamEntry.java | 3 +- .../ozone/client/protocol/ClientProtocol.java | 16 + .../apache/hadoop/ozone/client/rpc/RpcClient.java | 54 +- hadoop-ozone/common/pom.xml | 4 + .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 7 + .../ozone/om/ha/OMFailoverProxyProvider.java | 55 +- .../hadoop/ozone/om/helpers/BucketLayout.java | 62 ++ .../hadoop/ozone/om/helpers}/OMNodeDetails.java | 86 +- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 56 +- .../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 16 +- .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 1 - .../ozone/om/helpers/OmKeyLocationInfoGroup.java | 16 +- .../hadoop/ozone/om/helpers/OzoneFSUtils.java | 25 +- .../hadoop/ozone/om/helpers/WithObjectID.java | 6 +- .../ozone/om/protocol/OMInterServiceProtocol.java} | 28 +- .../ozone/om/protocolPB/Hadoop3OmTransport.java | 2 +- .../OMInterServiceProtocolClientSideImpl.java | 121 +++ .../om/protocolPB/OMInterServiceProtocolPB.java | 52 +- ...OzoneManagerProtocolClientSideTranslatorPB.java | 1 + .../apache/hadoop/ozone/util/OzoneVersionInfo.java | 5 +- .../main/resources/ozone-version-info.properties | 3 +- .../ozone/om/ha/TestOMFailoverProxyProvider.java | 7 +- .../om/helpers/TestOmKeyLocationInfoGroup.java | 22 +- hadoop-ozone/dev-support/checks/_lib.sh | 9 +- hadoop-ozone/dev-support/docker/Dockerfile | 38 +- hadoop-ozone/dev-support/intellij/ozone-site.xml | 2 +- .../dist/src/main/compose/ozone-ha/docker-config | 1 + .../{ozonesecure-mr => ozone-mr/hadoop33}/.env | 7 +- .../compose/ozone-mr/hadoop33/docker-compose.yaml | 110 +++ .../run.sh => ozone-mr/hadoop33/docker-config} | 11 +- .../{ozone/run.sh => ozone-mr/hadoop33/test.sh} | 26 +- .../src/main/compose/ozone-om-ha/docker-config | 1 + hadoop-ozone/dist/src/main/compose/ozone/README.md | 6 + .../src/main/compose/ozone/docker-compose.yaml | 2 +- .../dist/src/main/compose/ozone/docker-config | 1 + hadoop-ozone/dist/src/main/compose/ozone/run.sh | 4 + .../src/main/compose/ozonesecure-ha/docker-config | 4 + .../dist/src/main/compose/ozonesecure-mr/.env | 2 +- .../src/main/compose/ozonesecure/docker-config | 6 +- hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 11 +- hadoop-ozone/dist/src/main/license/jar-report.txt | 46 +- .../dist/src/main/license/update-jar-report.sh | 2 +- .../src/main/smoketest/admincli/container.robot | 4 - .../src/main/smoketest/admincli/datanode.robot | 18 + .../dist/src/main/smoketest/basic/links.robot | 10 +- .../src/main/smoketest/ozonefs/hadoopo3fs.robot | 2 +- .../dist/src/main/smoketest/s3/boto3.robot | 5 +- .../dist/src/main/smoketest/s3/boto_client.py | 10 +- .../dist/src/main/smoketest/s3/bucketcreate.robot | 6 +- .../dist/src/main/smoketest/s3/bucketdelete.robot | 3 +- .../dist/src/main/smoketest/s3/buckethead.robot | 3 +- .../dist/src/main/smoketest/s3/commonawslib.robot | 10 +- .../src/main/smoketest/s3/s3_compatbility_check.sh | 39 + .../dist/src/shell/ozone/ozone-functions.sh | 2 +- .../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 9 +- .../apache/hadoop/ozone/insight/LogSubcommand.java | 9 +- hadoop-ozone/integration-test/pom.xml | 5 + .../hadoop/fs/ozone/TestOzoneFileSystem.java | 12 +- .../fs/ozone/TestOzoneFileSystemWithFSO.java | 3 +- .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 55 +- .../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 30 +- .../ozone/contract/ITestOzoneContractDistCp.java | 8 + .../rooted/ITestRootedOzoneContractDistCp.java | 8 + .../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 10 +- .../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 6 +- .../TestContainerStateManagerIntegration.java | 40 +- .../hdds/scm/pipeline/TestMultiRaftSetup.java | 173 ++++ .../hdds/scm/pipeline/TestPipelineClose.java | 2 + .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 24 +- .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 5 + .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 7 +- .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 235 ++++- .../hadoop/ozone/MiniOzoneOMHAClusterImpl.java | 5 +- .../org/apache/hadoop/ozone/OzoneTestUtils.java | 17 + .../ozone/TestContainerBalancerOperations.java | 11 +- .../java/org/apache/hadoop/ozone/TestDataUtil.java | 40 +- .../hadoop/ozone/TestOzoneConfigurationFields.java | 3 + .../hadoop/ozone/TestStorageContainerManager.java | 12 +- .../ozone/client/rpc/TestBlockOutputStream.java | 13 +- .../rpc/TestBlockOutputStreamFlushDelay.java | 13 +- .../rpc/TestBlockOutputStreamWithFailures.java | 17 +- ...estBlockOutputStreamWithFailuresFlushDelay.java | 17 +- .../client/rpc/TestOzoneAtRestEncryption.java | 130 ++- .../rpc/TestOzoneClientMultipartUploadWithFSO.java | 13 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 68 +- .../ozone/client/rpc/TestWatchForCommit.java | 3 +- .../ozone/container/TestContainerReplication.java | 82 +- .../apache/hadoop/ozone/container/TestHelper.java | 2 +- .../commandhandler/TestBlockDeletion.java | 369 ++++--- .../server/TestSecureContainerServer.java | 6 + .../hadoop/ozone/om/TestOMRatisSnapshots.java | 13 +- .../hadoop/ozone/om/TestOMStartupWithLayout.java | 197 ---- .../hadoop/ozone/om/TestObjectStoreWithFSO.java | 76 +- .../hadoop/ozone/om/TestOmBlockVersioning.java | 21 +- .../hadoop/ozone/om/TestOzoneManagerBootstrap.java | 216 +++++ .../ozone/om/TestOzoneManagerConfiguration.java | 2 +- .../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 3 + .../ozone/recon/TestReconWithOzoneManagerFSO.java | 166 ++++ .../hadoop/ozone/scm/TestCloseContainer.java | 8 + .../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 99 +- .../TestSCMContainerPlacementPolicyMetrics.java | 2 +- .../ozone/scm/TestSCMInstallSnapshotWithHA.java | 19 +- .../scm/node/TestDecommissionAndMaintenance.java | 6 +- .../hadoop/ozone/shell/TestOzoneShellHA.java | 9 +- .../dev-support/findbugsExcludeFile.xml | 6 + hadoop-ozone/interface-client/pom.xml | 66 +- .../src/main/proto/OmClientProtocol.proto | 10 + .../src/main/proto/OmInterServiceProtocol.proto | 64 ++ hadoop-ozone/interface-storage/pom.xml | 2 +- .../apache/hadoop/ozone/om/OMMetadataManager.java | 7 + .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 45 +- .../apache/hadoop/ozone/om/OMPolicyProvider.java | 4 + .../apache/hadoop/ozone/om/OMStarterInterface.java | 2 + .../hadoop/ozone/om/OmMetadataManagerImpl.java | 5 + .../org/apache/hadoop/ozone/om/OzoneManager.java | 416 ++++++-- .../hadoop/ozone/om/OzoneManagerStarter.java | 47 +- .../hadoop/ozone/om/TrashOzoneFileSystem.java | 4 +- .../apache/hadoop/ozone/om/TrashPolicyOzone.java | 13 +- .../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 11 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 383 +++++--- .../ozone/om/ratis/OzoneManagerStateMachine.java | 30 + .../om/ratis/utils/OzoneManagerRatisUtils.java | 88 ++ .../om/request/bucket/OMBucketCreateRequest.java | 102 +- .../om/request/bucket/acl/OMBucketAclRequest.java | 8 +- .../request/bucket/acl/OMBucketAddAclRequest.java | 7 - .../bucket/acl/OMBucketRemoveAclRequest.java | 7 - .../request/bucket/acl/OMBucketSetAclRequest.java | 7 - .../hadoop/ozone/om/request/key/OMKeyRequest.java | 24 +- .../S3MultipartUploadCommitPartRequest.java | 11 +- .../request/s3/security/S3RevokeSecretRequest.java | 2 +- .../om/snapshot/OzoneManagerSnapshotProvider.java | 18 +- .../ozone/om/upgrade/OMUpgradeFinalizer.java | 3 +- .../OMInterServiceProtocolServerSideImpl.java | 90 ++ ...OzoneManagerProtocolServerSideTranslatorPB.java | 76 +- .../protocolPB/OzoneManagerRequestHandler.java | 8 +- .../hadoop/ozone/om/TestOzoneManagerStarter.java | 6 + .../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +- .../om/ratis/TestOzoneManagerRatisServer.java | 8 +- .../TestS3MultipartUploadCompleteRequest.java | 19 +- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 4 +- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 5 +- .../ozone/BasicRootedOzoneClientAdapterImpl.java | 40 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 9 +- .../hadoop/fs/ozone/Hadoop27RpcTransport.java | 2 +- .../hadoop/ozone/recon/ReconControllerModule.java | 2 + .../org/apache/hadoop/ozone/recon/ReconServer.java | 11 +- .../org/apache/hadoop/ozone/recon/ReconUtils.java | 31 + .../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 1019 ++++++++++++++++++++ .../hadoop/ozone/recon/api/NodeEndpoint.java | 10 +- .../hadoop/ozone/recon/api/types/DUResponse.java | 177 ++++ .../hadoop/ozone/recon/api/types/EntityType.java | 16 +- .../api/types/FileSizeDistributionResponse.java | 61 ++ .../hadoop/ozone/recon/api/types/NSSummary.java | 57 +- .../recon/api/types/NamespaceSummaryResponse.java | 106 ++ .../ozone/recon/api/types/QuotaUsageResponse.java | 69 ++ .../ozone/recon/api/types/ResponseStatus.java | 13 +- .../hadoop/ozone/recon/codec/NSSummaryCodec.java | 56 +- .../hadoop/ozone/recon/scm/ReconNodeManager.java | 106 +- .../ozone/recon/scm/ReconPipelineManager.java | 4 +- .../recon/scm/ReconPipelineReportHandler.java | 22 +- .../recon/spi/ReconNamespaceSummaryManager.java | 4 +- .../spi/impl/ReconNamespaceSummaryManagerImpl.java | 7 +- .../ozone/recon/tasks/FileSizeCountTask.java | 24 +- .../hadoop/ozone/recon/tasks/NSSummaryTask.java | 302 ++++++ .../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 1 + .../ozone/recon/tasks/OMDBUpdatesHandler.java | 2 +- .../webapps/recon/ozone-recon-web/api/db.json | 324 ++++++- .../webapps/recon/ozone-recon-web/api/routes.json | 22 +- .../src/components/navBar/navBar.tsx | 5 + .../src/components/rightDrawer/rightDrawer.tsx | 72 ++ .../src/constants/breadcrumbs.constants.tsx | 3 +- .../webapps/recon/ozone-recon-web/src/routes.tsx | 5 + .../src/views/datanodes/datanodes.tsx | 8 +- .../diskUsage/diskUsage.less} | 41 +- .../src/views/diskUsage/diskUsage.tsx | 439 +++++++++ .../src/views/overview/overview.tsx | 3 +- .../ozone/recon/OMMetadataManagerTestUtils.java | 109 +++ .../ozone/recon/api/TestNSSummaryEndpoint.java | 648 +++++++++++++ .../ozone/recon/fsck/TestContainerHealthTask.java | 3 +- .../ozone/recon/scm/TestReconNodeManager.java | 38 + .../impl/TestReconNamespaceSummaryManagerImpl.java | 22 +- .../recon/tasks/TestContainerKeyMapperTask.java | 17 +- .../ozone/recon/tasks/TestNSSummaryTask.java | 519 ++++++++++ .../java/org/apache/hadoop/ozone/s3/Gateway.java | 40 + .../hadoop/ozone/s3/OzoneClientProducer.java | 4 +- .../hadoop/ozone/s3/S3GatewayConfigKeys.java | 7 + .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 7 +- .../apache/hadoop/ozone/s3/util/ContinueToken.java | 15 +- .../hadoop/ozone/client/OzoneBucketStub.java | 20 + .../hadoop/ozone/s3/util/TestContinueToken.java | 20 + .../ozone/audit/parser/common/DatabaseHelper.java | 4 +- .../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 1 + .../hadoop/ozone/freon/BaseFreonGenerator.java | 12 +- .../freon/FollowerAppendLogEntryGenerator.java | 4 +- .../hadoop/ozone/freon/RandomKeyGenerator.java | 19 +- .../hadoop/ozone/freon/StreamingGenerator.java | 70 +- .../containergenerator/GeneratorDatanode.java | 60 +- .../ozone/genesis/BenchMarkContainerStateMap.java | 6 +- .../ozone/shell/bucket/CreateBucketHandler.java | 20 +- .../ozone/shell/bucket/InfoBucketHandler.java | 52 +- .../hadoop/ozone/shell/keys/CopyKeyHandler.java | 4 +- .../hadoop/ozone/shell/keys/PutKeyHandler.java | 4 +- .../containergenerator/TestGeneratorDatanode.java | 71 ++ pom.xml | 19 +- 405 files changed, 17159 insertions(+), 5676 deletions(-) diff --cc hadoop-ozone/interface-client/pom.xml index 1695e4e,743dcb0..e6325ca --- a/hadoop-ozone/interface-client/pom.xml +++ b/hadoop-ozone/interface-client/pom.xml @@@ -73,26 -55,74 +73,82 @@@ https://maven.apache.org/xsd/maven-4.0. <goals> <goal>compile</goal> <goal>test-compile</goal> + <goal>compile-custom</goal> + <goal>test-compile-custom</goal> </goals> <configuration> - <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot> <protocArtifact> - com.google.protobuf:protoc:${hadooprpc.protobuf.version}:exe:${os.detected.classifier} + com.google.protobuf:protoc:${proto2.hadooprpc.protobuf.version}:exe:${os.detected.classifier} </protocArtifact> + <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot> - <includes> - <include>OmClientProtocol.proto</include> - <include>Security.proto</include> - </includes> + <outputDirectory>target/generated-sources/protobuf/java</outputDirectory> + <clearOutputDirectory>false</clearOutputDirectory> + <pluginId>grpc-java</pluginId> + <pluginArtifact> + io.grpc:protoc-gen-grpc-java:${io.grpc.version}:exe:${os.detected.classifier} + </pluginArtifact> </configuration> </execution> + <execution> + <id>compile-protoc3</id> + <goals> + <goal>compile</goal> + <goal>test-compile</goal> + </goals> + <configuration> + <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot> + <protocArtifact> + com.google.protobuf:protoc:${proto3.hadooprpc.protobuf.version}:exe:${os.detected.classifier} + </protocArtifact> + <outputDirectory>target/generated-sources/protobuf/java/proto3</outputDirectory> + <clearOutputDirectory>false</clearOutputDirectory> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <phase>generate-sources</phase> + <configuration> + <tasks> + <replace token="com.google.protobuf" + value="org.apache.hadoop.thirdparty.protobuf" + dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto"> + </replace> + <replace token="com.google.protobuf" + value="org.apache.hadoop.thirdparty.protobuf" + dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/security/proto"> + </replace> + <replace token="org.apache.hadoop.ozone.protocol.proto" + value="org.apache.hadoop.ozone.protocol.proto3" + dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto"> + </replace> + <replace token="org.apache.hadoop.ozone.security.proto" + value="org.apache.hadoop.ozone.security.proto3" + dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/security/proto"> + </replace> + <replace token="org.apache.hadoop.hdds.protocol.proto" + value="org.apache.hadoop.hdds.protocol.proto3" + dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto"> + </replace> + <replace token="org.apache.hadoop.ozone.security.proto" + value="org.apache.hadoop.ozone.security.proto3" + dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto"> + </replace> + <move file="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto" + tofile="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto3"/> + <move file="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/security/proto" + tofile="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/security/proto3"/> + <move file="target/generated-sources/protobuf/java/proto3" + tofile="target/generated-sources/protobuf/java/"/> + </tasks> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> </executions> </plugin> <plugin> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
