This is an automated email from the ASF dual-hosted git repository. adoroszlai pushed a commit to branch HDDS-6030 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit c48b559ce8ef14683e42942822ab0c6f8f769e46 Merge: c1790ce 634d666 Author: Doroszlai, Attila <[email protected]> AuthorDate: Tue Feb 1 17:33:30 2022 +0100 Merge remote-tracking branch 'origin/master' into HDDS-6030 CONTRIBUTING.md | 3 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 4 +- .../hadoop/hdds/conf/OzoneConfiguration.java | 4 +- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 +- .../apache/hadoop/hdds/scm/client/ScmClient.java | 20 + .../hdds/scm/container/ContainerReplicaInfo.java | 129 ++++ .../scm/container/ReplicationManagerReport.java | 95 ++- .../protocol/StorageContainerLocationProtocol.java | 18 + .../org/apache/hadoop/ozone/audit/SCMAction.java | 3 +- .../common/src/main/resources/ozone-default.xml | 6 +- .../scm/container/TestContainerReplicaInfo.java | 59 ++ .../container/TestReplicationManagerReport.java | 49 ++ .../hadoop/hdds/scm/container/package-info.java | 21 + ...TestUpgradeUtils.java => UpgradeTestUtils.java} | 4 +- .../common/helpers/DatanodeVersionFile.java | 8 +- .../ozone/container/common/impl/ContainerData.java | 19 +- .../container/common/impl/ContainerDataYaml.java | 11 +- ...OutVersion.java => ContainerLayoutVersion.java} | 34 +- .../container/keyvalue/KeyValueContainerCheck.java | 10 +- .../container/keyvalue/KeyValueContainerData.java | 8 +- .../ozone/container/keyvalue/KeyValueHandler.java | 6 +- .../keyvalue/impl/ChunkManagerDispatcher.java | 18 +- .../keyvalue/impl/FilePerBlockStrategy.java | 4 +- .../keyvalue/impl/FilePerChunkStrategy.java | 4 +- .../replication/GrpcReplicationClient.java | 15 +- .../ozone/container/common/ContainerTestUtils.java | 4 +- .../container/common/TestBlockDeletingService.java | 13 +- ...ersion.java => TestContainerLayoutVersion.java} | 12 +- .../common/TestKeyValueContainerData.java | 10 +- .../common/impl/TestContainerDataYaml.java | 16 +- .../impl/TestContainerDeletionChoosingPolicy.java | 8 +- .../common/impl/TestContainerPersistence.java | 8 +- .../container/common/impl/TestContainerSet.java | 8 +- .../container/common/impl/TestHddsDispatcher.java | 8 +- .../TestCloseContainerCommandHandler.java | 10 +- .../common/volume/TestStorageVolumeChecker.java | 8 +- ...tTestInfo.java => ContainerLayoutTestInfo.java} | 24 +- .../keyvalue/TestKeyValueBlockIterator.java | 10 +- .../container/keyvalue/TestKeyValueContainer.java | 16 +- .../keyvalue/TestKeyValueContainerCheck.java | 9 +- .../TestKeyValueContainerMarkUnhealthy.java | 8 +- .../container/keyvalue/TestKeyValueHandler.java | 8 +- .../container/keyvalue/TestTarContainerPacker.java | 8 +- .../keyvalue/impl/AbstractTestChunkManager.java | 8 +- .../keyvalue/impl/TestBlockManagerImpl.java | 10 +- .../keyvalue/impl/TestChunkManagerDummyImpl.java | 6 +- .../keyvalue/impl/TestFilePerBlockStrategy.java | 6 +- .../keyvalue/impl/TestFilePerChunkStrategy.java | 12 +- .../container/ozoneimpl/TestContainerReader.java | 8 +- .../container/ozoneimpl/TestOzoneContainer.java | 10 +- .../replication/TestReplicationSupervisor.java | 10 +- .../upgrade/TestDataNodeStartupSlvLessThanMlv.java | 4 +- hadoop-hdds/docs/content/feature/Observability.md | 2 +- .../{Observability.md => Observability.zh.md} | 63 +- hadoop-hdds/docs/content/tools/TestTools.md | 129 +--- hadoop-hdds/docs/content/tools/TestTools.zh.md | 129 +--- hadoop-hdds/docs/content/tools/_index.md | 1 - hadoop-hdds/docs/content/tools/_index.zh.md | 1 - .../themes/ozonedoc/layouts/shortcodes/image.html | 2 +- ...inerLocationProtocolClientSideTranslatorPB.java | 38 ++ .../org/apache/hadoop/hdds/utils/db/DBStore.java | 10 + .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 11 + .../apache/hadoop/hdds/utils/db/TestRDBStore.java | 24 + .../src/main/proto/ScmAdminProtocol.proto | 23 + .../interface-client/src/main/proto/hdds.proto | 25 + hadoop-hdds/server-scm/pom.xml | 10 - .../hdds/scm/container/ReplicationManager.java | 5 + .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 3 +- ...inerLocationProtocolServerSideTranslatorPB.java | 33 + .../hdds/scm/server/SCMClientProtocolServer.java | 32 + .../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 692 +++++++++++++++++++- .../java/org/apache/hadoop/hdds/scm/TestUtils.java | 699 --------------------- .../hadoop/hdds/scm/block/TestBlockManager.java | 18 +- .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 4 +- .../command/TestCommandStatusReportHandler.java | 4 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 6 +- .../container/TestCloseContainerEventHandler.java | 4 +- .../scm/container/TestContainerReportHandler.java | 4 +- .../TestIncrementalContainerReportHandler.java | 4 +- .../hdds/scm/container/TestReplicationManager.java | 4 +- .../scm/container/TestUnknownContainerReport.java | 2 +- .../algorithms/TestContainerPlacementFactory.java | 12 +- .../TestSCMContainerPlacementCapacity.java | 12 +- .../TestSCMContainerPlacementRackAware.java | 22 +- .../TestSCMContainerPlacementRandom.java | 16 +- .../hdds/scm/crl/TestCRLStatusReportHandler.java | 4 +- .../hdds/scm/node/TestContainerPlacement.java | 6 +- .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 43 +- .../hdds/scm/node/TestNodeDecommissionManager.java | 4 +- .../hdds/scm/node/TestNodeReportHandler.java | 10 +- .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 78 ++- .../hdds/scm/node/TestSCMNodeStorageStatMap.java | 15 +- .../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 16 +- .../hdds/scm/pipeline/TestPipelineManagerImpl.java | 11 +- .../scm/server/TestSCMBlockProtocolServer.java | 4 +- .../TestSCMHAUnfinalizedStateValidationAction.java | 5 +- .../scm/upgrade/TestScmStartupSlvLessThanMlv.java | 4 +- .../ozone/container/common/TestEndPoint.java | 23 +- .../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 11 +- hadoop-hdds/tools/pom.xml | 9 + .../hdds/scm/cli/ContainerOperationClient.java | 28 + .../hdds/scm/cli/container/ContainerCommands.java | 3 +- .../hdds/scm/cli/container/InfoSubcommand.java | 61 +- .../hdds/scm/cli/container/ReportSubcommand.java | 116 ++++ .../hdds/scm/cli/container/TestInfoSubCommand.java | 249 ++++++++ .../scm/cli/container/TestReportSubCommand.java | 159 +++++ .../apache/hadoop/ozone/client/OzoneBucket.java | 11 + .../checksum/AbstractBlockChecksumComputer.java | 43 ++ .../{ => checksum}/BaseFileChecksumHelper.java | 4 +- .../checksum/ReplicatedBlockChecksumComputer.java | 72 +++ .../ReplicatedFileChecksumHelper.java | 30 +- .../hadoop/ozone/client/checksum/package-info.java | 23 + .../ozone/client/protocol/ClientProtocol.java | 27 + .../apache/hadoop/ozone/client/rpc/RpcClient.java | 92 ++- .../TestReplicatedBlockChecksumComputer.java | 68 ++ .../TestReplicatedFileChecksumHelper.java | 129 +++- .../hadoop/ozone/client/checksum/package-info.java | 23 + .../hadoop/ozone/om/helpers/OmBucketArgs.java | 36 +- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 10 +- .../ozone/om/protocol/OzoneManagerProtocol.java | 12 + ...OzoneManagerProtocolClientSideTranslatorPB.java | 22 + hadoop-ozone/dev-support/checks/coverage.sh | 1 - hadoop-ozone/dist/pom.xml | 2 +- .../dist/src/main/compose/ozone-ha/docker-config | 4 + .../dist/src/main/compose/ozonesecure-mr/test.sh | 4 +- .../src/main/compose/ozonesecure/docker-config | 1 + .../main/compose/upgrade/compose/ha/docker-config | 4 +- .../compose/upgrade/compose/non-ha/docker-config | 2 +- hadoop-ozone/dist/src/main/docker/Dockerfile | 1 - .../dist/src/main/dockerlibexec/entrypoint.sh | 2 +- hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 5 +- hadoop-ozone/dist/src/shell/ozone/ozone | 17 - .../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 39 +- hadoop-ozone/insight/pom.xml | 1 - hadoop-ozone/integration-test/pom.xml | 10 - .../hadoop/fs/ozone/TestOzoneFileSystem.java | 2 + .../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 4 +- .../metrics/TestSCMContainerManagerMetrics.java | 2 + .../hdds/scm/pipeline/TestMultiRaftSetup.java | 1 + .../hdds/scm/pipeline/TestPipelineClose.java | 4 +- .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 2 + .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 2 +- .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 22 +- .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 374 +++++++---- .../hadoop/ozone/MiniOzoneOMHAClusterImpl.java | 116 ---- .../org/apache/hadoop/ozone/OzoneTestUtils.java | 23 + .../java/org/apache/hadoop/ozone/TestDataUtil.java | 2 +- .../hadoop/ozone/TestMiniOzoneOMHACluster.java | 6 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 35 +- .../hadoop/ozone/TestStorageContainerManager.java | 11 +- .../apache/hadoop/ozone/client/rpc/TestBCSID.java | 7 +- .../client/rpc/TestContainerStateMachine.java | 5 +- .../rpc/TestContainerStateMachineFailures.java | 161 ++++- .../rpc/TestContainerStateMachineFlushDelay.java | 4 +- .../client/rpc/TestDeleteWithSlowFollower.java | 11 +- .../client/rpc/TestOzoneAtRestEncryption.java | 12 +- .../rpc/TestOzoneClientMultipartUploadWithFSO.java | 46 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 115 ++-- .../hadoop/ozone/client/rpc/TestReadRetries.java | 4 +- .../ozone/client/rpc/TestSecureOzoneRpcClient.java | 6 +- .../client/rpc/read/TestChunkInputStream.java | 4 +- .../ozone/client/rpc/read/TestInputStreamBase.java | 15 +- .../ozone/client/rpc/read/TestKeyInputStream.java | 4 +- .../apache/hadoop/ozone/container/TestHelper.java | 7 +- .../commandhandler/TestBlockDeletion.java | 1 + .../commandhandler/TestCloseContainerHandler.java | 4 +- .../commandhandler/TestDeleteContainerHandler.java | 3 +- .../container/metrics/TestContainerMetrics.java | 3 +- .../container/server/TestContainerServer.java | 3 +- .../server/TestSecureContainerServer.java | 3 +- .../ozone/dn/ratis/TestDnRatisLogParser.java | 2 + .../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 7 +- .../TestDatanodeHddsVolumeFailureDetection.java | 7 +- .../freon/TestHadoopDirTreeGeneratorWithFSO.java | 4 +- .../ozone/om/TestContainerReportWithKeys.java | 2 +- .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 26 +- .../org/apache/hadoop/ozone/om/TestKeyPurging.java | 2 +- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 6 +- .../hadoop/ozone/om/TestOMUpgradeFinalization.java | 6 +- .../hadoop/ozone/om/TestObjectStoreWithFSO.java | 8 +- .../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 4 +- .../ozone/om/TestOmStartupSlvLessThanMlv.java | 4 +- .../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 12 +- .../ozone/om/TestOzoneManagerHAMetadataOnly.java | 2 +- .../ozone/om/TestOzoneManagerHAWithData.java | 2 +- .../ozone/om/TestOzoneManagerHAWithFailover.java | 2 +- .../hadoop/ozone/om/TestOzoneManagerPrepare.java | 2 +- .../hadoop/ozone/om/TestRecursiveAclWithFSO.java | 4 +- .../om/ratis/TestOzoneManagerRatisRequest.java | 4 +- .../snapshot/TestOzoneManagerSnapshotProvider.java | 8 +- .../hadoop/ozone/recon/TestReconScmHASnapshot.java | 70 +++ .../hadoop/ozone/recon/TestReconScmSnapshot.java | 133 ++++ .../ozone/recon/TestReconWithOzoneManager.java | 4 +- .../ozone/recon/TestReconWithOzoneManagerFSO.java | 6 +- .../ozone/recon/TestReconWithOzoneManagerHA.java | 8 +- .../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 2 +- .../ozone/scm/TestSCMInstallSnapshotWithHA.java | 18 +- .../ozone/scm/TestStorageContainerManagerHA.java | 4 +- .../scm/node/TestDecommissionAndMaintenance.java | 6 +- .../hadoop/ozone/shell/TestNSSummaryAdmin.java | 4 +- .../hadoop/ozone/shell/TestOzoneShellHA.java | 4 +- .../src/main/proto/OmClientProtocol.proto | 4 +- .../hadoop/ozone/om/codec/TestOmKeyInfoCodec.java | 4 +- .../ozone/om/codec/TestRepeatedOmKeyInfoCodec.java | 4 +- .../ozone/om/helpers/TestInstanceHelper.java | 68 -- .../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 39 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 6 +- .../om/ratis/utils/OzoneManagerRatisUtils.java | 9 +- .../om/request/bucket/OMBucketSetOwnerRequest.java | 199 ++++++ .../key/OMOpenKeysDeleteRequest.java | 4 +- .../response/bucket/OMBucketSetOwnerResponse.java | 80 +++ .../hadoop/ozone/om/TestBucketManagerImpl.java | 6 +- .../hadoop/ozone/om/TestKeyDeletingService.java | 31 +- .../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 16 +- .../hadoop/ozone/om/TestOmMetadataManager.java | 36 +- .../apache/hadoop/ozone/om/TestTrashService.java | 6 +- ...TestOzoneManagerDoubleBufferWithOMResponse.java | 8 +- ...OMRequestUtils.java => OMRequestTestUtils.java} | 10 +- .../request/TestOMClientRequestWithUserInfo.java | 2 +- .../request/bucket/TestOMBucketCreateRequest.java | 8 +- .../bucket/TestOMBucketCreateRequestWithFSO.java | 4 +- .../request/bucket/TestOMBucketDeleteRequest.java | 6 +- .../bucket/TestOMBucketSetPropertyRequest.java | 8 +- .../bucket/acl/TestOMBucketAddAclRequest.java | 12 +- .../bucket/acl/TestOMBucketRemoveAclRequest.java | 14 +- .../bucket/acl/TestOMBucketSetAclRequest.java | 12 +- .../request/file/TestOMDirectoryCreateRequest.java | 22 +- .../file/TestOMDirectoryCreateRequestWithFSO.java | 48 +- .../om/request/file/TestOMFileCreateRequest.java | 30 +- .../file/TestOMFileCreateRequestWithFSO.java | 22 +- .../om/request/key/TestOMAllocateBlockRequest.java | 12 +- .../key/TestOMAllocateBlockRequestWithFSO.java | 10 +- .../ozone/om/request/key/TestOMKeyAclRequest.java | 10 +- .../om/request/key/TestOMKeyAclRequestWithFSO.java | 8 +- .../om/request/key/TestOMKeyCommitRequest.java | 12 +- .../request/key/TestOMKeyCommitRequestWithFSO.java | 8 +- .../om/request/key/TestOMKeyCreateRequest.java | 8 +- .../request/key/TestOMKeyCreateRequestWithFSO.java | 6 +- .../om/request/key/TestOMKeyDeleteRequest.java | 10 +- .../request/key/TestOMKeyDeleteRequestWithFSO.java | 10 +- .../key/TestOMKeyPurgeRequestAndResponse.java | 8 +- .../om/request/key/TestOMKeyRenameRequest.java | 15 +- .../om/request/key/TestOMKeysDeleteRequest.java | 6 +- .../om/request/key/TestOMKeysRenameRequest.java | 6 +- .../request/key/TestOMOpenKeysDeleteRequest.java | 11 +- .../om/request/key/TestOMPrefixAclRequest.java | 6 +- .../TestS3InitiateMultipartUploadRequest.java | 6 +- ...estS3InitiateMultipartUploadRequestWithFSO.java | 4 +- .../s3/multipart/TestS3MultipartRequest.java | 12 +- .../TestS3MultipartUploadAbortRequest.java | 8 +- .../TestS3MultipartUploadAbortRequestWithFSO.java | 4 +- .../TestS3MultipartUploadCommitPartRequest.java | 12 +- ...tS3MultipartUploadCommitPartRequestWithFSO.java | 10 +- .../TestS3MultipartUploadCompleteRequest.java | 12 +- ...estS3MultipartUploadCompleteRequestWithFSO.java | 8 +- .../upgrade/TestOMCancelPrepareRequest.java | 2 +- .../request/volume/TestOMVolumeCreateRequest.java | 4 +- .../request/volume/TestOMVolumeDeleteRequest.java | 12 +- .../volume/TestOMVolumeSetOwnerRequest.java | 27 +- .../volume/TestOMVolumeSetQuotaRequest.java | 27 +- .../volume/acl/TestOMVolumeAddAclRequest.java | 12 +- .../volume/acl/TestOMVolumeRemoveAclRequest.java | 14 +- .../volume/acl/TestOMVolumeSetAclRequest.java | 12 +- .../file/TestOMDirectoryCreateResponse.java | 4 +- .../file/TestOMDirectoryCreateResponseWithFSO.java | 4 +- .../file/TestOMFileCreateResponseWithFSO.java | 4 +- .../response/key/TestOMAllocateBlockResponse.java | 4 +- .../key/TestOMAllocateBlockResponseWithFSO.java | 4 +- .../om/response/key/TestOMKeyCommitResponse.java | 6 +- .../key/TestOMKeyCommitResponseWithFSO.java | 8 +- .../key/TestOMKeyCreateResponseWithFSO.java | 4 +- .../om/response/key/TestOMKeyDeleteResponse.java | 4 +- .../key/TestOMKeyDeleteResponseWithFSO.java | 12 +- .../om/response/key/TestOMKeyRenameResponse.java | 14 +- .../ozone/om/response/key/TestOMKeyResponse.java | 4 +- .../om/response/key/TestOMKeysDeleteResponse.java | 4 +- .../om/response/key/TestOMKeysRenameResponse.java | 8 +- .../response/key/TestOMOpenKeysDeleteResponse.java | 8 +- ...S3MultipartUploadCommitPartResponseWithFSO.java | 10 +- ...stS3MultipartUploadCompleteResponseWithFSO.java | 18 +- .../security/acl/TestOzoneNativeAuthorizer.java | 6 +- .../hadoop/ozone/security/acl/TestParentAcl.java | 6 +- .../hadoop/ozone/security/acl/TestVolumeOwner.java | 6 +- .../hadoop/ozone/recon/scm/ReconNodeManager.java | 6 + .../impl/StorageContainerServiceProviderImpl.java | 48 +- .../hadoop/ozone/client/OzoneBucketStub.java | 2 +- .../hadoop/ozone/s3/endpoint/TestObjectHead.java | 2 +- .../tools/dev-support/findbugsExcludeFile.xml | 4 - hadoop-ozone/tools/pom.xml | 15 - .../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 6 +- .../apache/hadoop/ozone/debug/ReadReplicas.java | 247 ++++++++ .../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 8 +- .../containergenerator/GeneratorDatanode.java | 6 +- .../hadoop/ozone/genesis/BenchMarkCRCBatch.java | 141 ----- .../ozone/genesis/BenchMarkCRCStreaming.java | 173 ----- .../ozone/genesis/BenchMarkContainerStateMap.java | 199 ------ .../ozone/genesis/BenchMarkDatanodeDispatcher.java | 339 ---------- .../apache/hadoop/ozone/genesis/BenchMarkSCM.java | 126 ---- .../ozone/genesis/BenchmarkBlockDataToString.java | 166 ----- .../ozone/genesis/BenchmarkChunkManager.java | 180 ------ .../org/apache/hadoop/ozone/genesis/Genesis.java | 107 ---- .../ozone/genesis/GenesisMemoryProfiler.java | 61 -- .../apache/hadoop/ozone/genesis/GenesisUtil.java | 162 ----- .../apache/hadoop/ozone/genesis/package-info.java | 25 - .../hadoop/ozone/shell/bucket/BucketCommands.java | 3 +- .../ozone/shell/bucket/UpdateBucketHandler.java | 62 ++ pom.xml | 13 - 307 files changed, 4894 insertions(+), 4210 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
