This is an automated email from the ASF dual-hosted git repository. erose pushed a commit to branch HDDS-10239-container-reconciliation in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 8676000a6f6a1fe50b6d6764447826bf58a85548 Merge: d585363a7a 2eed61cdfa Author: Ethan Rose <[email protected]> AuthorDate: Tue Jul 23 12:10:43 2024 -0400 Merge branch 'master' into HDDS-10239-container-reconciliation Conflicts: hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java .github/workflows/build-ratis.yml | 12 +- .github/workflows/ci-with-ratis.yml | 53 + .github/workflows/ci.yml | 116 +- .github/workflows/intermittent-test-check.yml | 10 +- .github/workflows/populate-cache.yml | 24 +- .github/workflows/repeat-acceptance.yml | 4 +- .gitignore | 1 + .mvn/{gradle-enterprise.xml => develocity.xml} | 15 +- .mvn/extensions.xml | 6 +- dev-support/ci/download-nodejs.sh | 63 + hadoop-hdds/annotations/pom.xml | 12 + hadoop-hdds/client/pom.xml | 37 + .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 92 +- .../hadoop/hdds/scm/XceiverClientMetrics.java | 59 +- .../apache/hadoop/hdds/scm/XceiverClientRatis.java | 106 +- .../hadoop/hdds/scm/client/HddsClientUtils.java | 55 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 31 +- .../hadoop/hdds/scm/storage/ChunkInputStream.java | 25 +- .../ozone/client/io/BlockInputStreamFactory.java | 3 +- .../client/io/BlockInputStreamFactoryImpl.java | 3 +- .../hadoop/ozone/client/io/ECBlockInputStream.java | 7 +- .../hdds/scm/storage/DummyBlockInputStream.java | 2 +- .../storage/DummyBlockInputStreamWithRetry.java | 2 +- .../hdds/scm/storage/TestBlockInputStream.java | 2 +- .../storage/TestBlockOutputStreamCorrectness.java | 10 +- .../client/io/TestBlockInputStreamFactoryImpl.java | 11 +- hadoop-hdds/common/pom.xml | 53 +- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 7 - .../java/org/apache/hadoop/hdds/StringUtils.java | 79 - .../org/apache/hadoop/hdds/client/BlockID.java | 57 +- .../hadoop/hdds/client/DecommissionUtils.java | 4 +- .../org/apache/hadoop/hdds/client/OzoneQuota.java | 7 +- .../hadoop/hdds/conf/DelegatingProperties.java | 190 + .../hadoop/hdds/conf/OzoneConfiguration.java | 49 + .../hadoop/hdds/fs/CachingSpaceUsageSource.java | 134 +- .../hdds/ratis/ContainerCommandRequestMessage.java | 4 + .../hadoop/hdds/ratis/conf/RatisClientConfig.java | 22 +- .../apache/hadoop/hdds/recon/ReconConfigKeys.java | 3 +- .../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 60 +- .../apache/hadoop/hdds/scm/client/ScmClient.java | 3 + .../hadoop/hdds/scm/exceptions/SCMException.java | 6 + .../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 15 +- .../protocol/StorageContainerLocationProtocol.java | 3 + .../hdds/scm/storage/ContainerProtocolCalls.java | 47 +- .../hadoop/hdds/security/SecurityConfig.java | 51 +- .../x509/certificate/utils/CertificateCodec.java | 107 +- .../certificate/utils/SelfSignedCertificate.java | 24 +- .../hadoop/hdds/security/x509/crl/CRLStatus.java | 87 - .../hdds/security/x509/crl/package-info.java | 24 - .../hadoop/hdds/utils/BackgroundService.java | 7 +- .../utils/LegacyHadoopConfigurationSource.java | 58 +- .../org/apache/hadoop/ozone/ClientVersion.java | 4 + .../java/org/apache/hadoop/ozone/OzoneConsts.java | 7 +- .../apache/hadoop/ozone/OzoneManagerVersion.java | 2 + .../org/apache/hadoop/ozone/OzoneSecurityUtil.java | 5 +- .../org/apache/hadoop/util/PerformanceMetrics.java | 23 +- .../hadoop/util/PerformanceMetricsInitializer.java | 24 +- .../services/javax.annotation.processing.Processor | 18 - .../common/src/main/resources/ozone-default.xml | 36 +- .../apache/hadoop/hdds/client/TestOzoneQuota.java} | 24 +- .../hadoop/hdds/conf/TestOzoneConfiguration.java | 62 + .../hdds/fs/TestCachingSpaceUsageSource.java | 72 +- .../ratis/TestContainerCommandRequestMessage.java | 3 + .../hdds/security/x509/CertificateTestUtils.java | 1 + .../ozone/container/ContainerTestHelper.java | 22 +- hadoop-hdds/config/pom.xml | 37 + hadoop-hdds/container-service/pom.xml | 37 + .../hdds/datanode/metadata/CRLDBDefinition.java | 100 - .../hdds/datanode/metadata/DatanodeCRLStore.java | 84 - .../datanode/metadata/DatanodeCRLStoreImpl.java | 128 - .../apache/hadoop/ozone/HddsDatanodeService.java | 24 +- .../container/common/helpers/ContainerMetrics.java | 61 +- .../container/common/impl/HddsDispatcher.java | 11 +- .../common/report/CRLStatusReportPublisher.java | 82 - .../common/report/ReportPublisherFactory.java | 2 - .../common/statemachine/DatanodeStateMachine.java | 12 +- .../common/statemachine/StateContext.java | 13 - .../common/transport/server/ratis/CSMMetrics.java | 64 +- .../server/ratis/ContainerStateMachine.java | 64 +- .../transport/server/ratis/DispatcherContext.java | 8 + .../transport/server/ratis/XceiverServerRatis.java | 26 +- .../ozone/container/keyvalue/KeyValueHandler.java | 23 +- .../container/keyvalue/helpers/BlockUtils.java | 23 +- .../container/keyvalue/impl/BlockManagerImpl.java | 20 +- .../keyvalue/interfaces/BlockManager.java | 4 +- .../metadata/TestDatanodeCRLStoreImpl.java | 117 - .../hdds/datanode/metadata/package-info.java | 22 - .../hadoop/ozone/TestHddsDatanodeService.java | 24 +- .../hadoop/ozone/TestHddsSecureDatanodeInit.java | 59 +- .../container/common/helpers/TestBlockData.java | 2 +- .../common/report/TestReportPublisher.java | 74 - .../common/report/TestReportPublisherFactory.java | 12 - .../common/statemachine/TestStateContext.java | 11 +- .../TestKeyValueHandlerWithUnhealthyContainer.java | 59 + .../impl/TestKeyValueStreamDataChannel.java | 4 +- hadoop-hdds/{annotations => crypto-api}/pom.xml | 34 +- .../{annotations => crypto-default}/pom.xml | 34 +- hadoop-hdds/docs/content/concept/Recon.zh.md | 2 +- .../content/concept/StorageContainerManager.md | 2 +- .../docs/content/feature/Nonrolling-Upgrade.md | 14 - hadoop-hdds/docs/content/start/OnPrem.md | 2 +- hadoop-hdds/docs/content/start/OnPrem.zh.md | 6 +- hadoop-hdds/docs/dev-support/bin/generate-site.sh | 2 +- hadoop-hdds/docs/pom.xml | 7 + hadoop-hdds/erasurecode/pom.xml | 7 + hadoop-hdds/framework/pom.xml | 50 + .../hdds/conf/DatanodeRatisServerConfig.java | 8 +- .../hadoop/hdds/protocol/SCMSecurityProtocol.java | 31 +- .../SCMSecurityProtocolClientSideTranslatorPB.java | 61 +- .../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 46 - ...inerLocationProtocolClientSideTranslatorPB.java | 15 + .../scm/update/client/CRLClientUpdateHandler.java | 202 - .../hadoop/hdds/scm/update/client/CRLStore.java | 34 - .../hdds/scm/update/client/ClientCRLStore.java | 98 - .../scm/update/client/ClientUpdateHandler.java | 29 - .../client/SCMUpdateClientConfiguration.java | 53 - .../update/client/SCMUpdateServiceGrpcClient.java | 219 - .../scm/update/client/UpdateServiceConfig.java | 43 - .../hdds/scm/update/client/package-info.java | 22 - .../hdds/scm/update/server/CRLClientInfo.java | 56 - .../scm/update/server/SCMUpdateClientInfo.java | 67 - .../hdds/scm/update/server/package-info.java | 22 - .../x509/certificate/authority/BaseApprover.java | 258 - .../x509/certificate/authority/CRLApprover.java | 43 - .../certificate/authority/CertificateApprover.java | 27 +- .../certificate/authority/CertificateServer.java | 46 +- .../certificate/authority/CertificateStore.java | 93 +- .../certificate/authority/DefaultApprover.java | 217 +- .../certificate/authority/DefaultCAServer.java | 164 +- .../certificate/authority/DefaultCRLApprover.java | 58 - .../client/DefaultCertificateClient.java | 2 +- .../certificate/client/SCMCertificateClient.java | 19 +- .../hadoop/hdds/security/x509/crl/CRLCodec.java | 220 - .../hadoop/hdds/security/x509/crl/CRLInfo.java | 226 - .../hadoop/hdds/server/http/HttpServer2.java | 20 +- .../apache/hadoop/hdds/utils/HddsServerUtil.java | 75 + .../org/apache/hadoop/hdds/utils/SignalLogger.java | 30 +- .../apache/hadoop/hdds/utils/TransactionInfo.java | 2 +- .../hadoop/hdds/utils/db/DBConfigFromFile.java | 2 +- .../apache/hadoop/hdds/utils/db/RocksDatabase.java | 2 +- .../x509/certificate/authority/MockApprover.java | 58 - .../x509/certificate/authority/MockCAStore.java | 56 +- .../certificate/authority/TestDefaultCAServer.java | 135 +- .../certificate/authority/TestDefaultProfile.java | 33 +- .../client/CertificateClientTestImpl.java | 64 +- .../client/TestDefaultCertificateClient.java | 12 +- .../client/TestDnCertificateClientInit.java | 7 +- .../client/TestRootCaRotationPoller.java | 15 +- .../x509/certificate/utils/TestCRLCodec.java | 288 - .../certificate/utils/TestCertificateCodec.java | 174 +- .../certificate/utils/TestRootCertificate.java | 69 +- hadoop-hdds/hadoop-dependency-client/pom.xml | 12 + hadoop-hdds/hadoop-dependency-server/pom.xml | 49 +- hadoop-hdds/hadoop-dependency-test/pom.xml | 12 + hadoop-hdds/interface-admin/pom.xml | 7 + .../src/main/proto/ScmAdminProtocol.proto | 41 +- hadoop-hdds/interface-client/pom.xml | 7 + hadoop-hdds/interface-server/pom.xml | 7 + .../src/main/proto/ScmServerProtocol.proto | 3 + .../src/main/proto/ScmServerSecurityProtocol.proto | 20 + hadoop-hdds/managed-rocksdb/pom.xml | 13 +- hadoop-hdds/pom.xml | 2 + hadoop-hdds/rocks-native/pom.xml | 12 +- .../src/main/patches/rocks-native.patch | 5 +- hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 7 + hadoop-hdds/server-scm/pom.xml | 42 + .../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 27 - .../balancer/AbstractFindTargetGreedy.java | 4 + .../scm/container/balancer/ContainerBalancer.java | 22 + .../balancer/ContainerBalancerConfiguration.java | 2 +- .../balancer/ContainerBalancerMetrics.java | 5 + .../ContainerBalancerSelectionCriteria.java | 18 +- .../balancer/ContainerBalancerStatusInfo.java | 54 + .../container/balancer/ContainerBalancerTask.java | 135 +- .../ContainerBalancerTaskIterationStatusInfo.java | 104 + .../container/balancer/ContainerMoveSelection.java | 22 + .../scm/container/balancer/FindSourceGreedy.java | 5 + .../scm/container/balancer/FindSourceStrategy.java | 3 + .../scm/container/balancer/FindTargetStrategy.java | 3 + .../algorithms/SCMContainerPlacementCapacity.java | 6 + .../algorithms/SCMContainerPlacementRandom.java | 6 + .../placement/metrics/SCMPerformanceMetrics.java | 94 + .../apache/hadoop/hdds/scm/events/SCMEvents.java | 10 - .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 19 +- .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 4 +- .../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 11 - .../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 45 - .../hdds/scm/metadata/SCMMetadataStoreImpl.java | 77 - .../hadoop/hdds/scm/node/NodeStateManager.java | 5 + .../hadoop/hdds/scm/node/SCMNodeManager.java | 8 +- .../hadoop/hdds/scm/node/states/NodeStateMap.java | 10 + .../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 31 +- .../algorithms/RoundRobinPipelineChoosePolicy.java | 63 + .../SCMSecurityProtocolServerSideTranslatorPB.java | 60 +- ...inerLocationProtocolServerSideTranslatorPB.java | 14 + .../apache/hadoop/hdds/scm/safemode/Precheck.java | 29 - .../hadoop/hdds/scm/safemode/SafeModePrecheck.java | 69 - .../hdds/scm/security/CRLStatusReportHandler.java | 87 - .../hdds/scm/security/RootCARotationManager.java | 36 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 8 + .../hadoop/hdds/scm/server/SCMCertStore.java | 205 +- .../hdds/scm/server/SCMClientProtocolServer.java | 75 +- .../scm/server/SCMDatanodeHeartbeatDispatcher.java | 14 - .../hdds/scm/server/SCMSecurityProtocolServer.java | 71 +- .../hdds/scm/server/StorageContainerManager.java | 58 +- .../scm/server/StorageContainerManagerStarter.java | 4 +- .../hadoop/hdds/scm/update/server/SCMCRLStore.java | 51 - .../scm/update/server/SCMCRLUpdateHandler.java | 142 - .../scm/update/server/SCMUpdateClientManager.java | 149 - .../hdds/scm/update/server/SCMUpdateHandler.java | 53 - .../update/server/SCMUpdateServiceGrpcServer.java | 100 - .../scm/update/server/SCMUpdateServiceImpl.java | 119 - .../hdds/scm/update/server/package-info.java | 22 - .../org/apache/hadoop/ozone/audit/SCMAction.java | 1 + .../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 16 - .../TestContainerBalancerDatanodeNodeLimit.java | 2 + .../balancer/TestContainerBalancerStatusInfo.java | 58 + .../TestSCMContainerPlacementCapacity.java | 3 +- .../TestSCMContainerPlacementRandom.java | 6 +- .../hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java | 63 +- .../hdds/scm/ha/io/TestX509CertificateCodec.java | 8 + .../scm/metadata/TestX509CertificateCodec.java | 8 + .../TestRoundRobinPipelineChoosePolicy.java | 170 + .../scm/security/TestCRLStatusReportHandler.java | 137 - .../scm/security/TestRootCARotationManager.java | 7 +- .../hadoop/hdds/scm/server/TestSCMCertStore.java | 198 +- .../hdds/scm/update/server/MockCRLStore.java | 140 - .../server/TestSCMUpdateServiceGrpcServer.java | 302 - .../ozone/container/common/TestEndPoint.java | 45 +- .../placement/TestContainerPlacement.java | 6 +- hadoop-hdds/test-utils/pom.xml | 11 + hadoop-hdds/tools/pom.xml | 47 + .../scm/cli/ContainerBalancerStatusSubcommand.java | 137 +- .../hdds/scm/cli/ContainerOperationClient.java | 6 + .../hadoop/hdds/scm/cli/cert/ListSubcommand.java | 15 +- .../datanode/TestContainerBalancerSubCommand.java | 198 +- hadoop-ozone/client/pom.xml | 42 + .../apache/hadoop/ozone/client/ObjectStore.java | 92 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 36 +- .../org/apache/hadoop/ozone/client/OzoneKey.java | 8 +- .../hadoop/ozone/client/OzoneKeyDetails.java | 30 +- .../client/checksum/ECFileChecksumHelper.java | 7 +- .../checksum/ReplicatedFileChecksumHelper.java | 8 +- .../ozone/client/io/ECBlockOutputStreamEntry.java | 4 +- .../hadoop/ozone/client/io/KeyInputStream.java | 8 +- .../ozone/client/protocol/ClientProtocol.java | 45 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 112 +- hadoop-ozone/common/pom.xml | 42 + .../main/java/org/apache/hadoop/ozone/OmUtils.java | 51 +- .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 5 + .../ozone/om/exceptions/OMNotLeaderException.java | 7 + .../apache/hadoop/ozone/om/helpers/ErrorInfo.java | 33 +- .../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 34 +- .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 40 + .../ozone/om/protocol/OzoneManagerProtocol.java | 26 +- ...OzoneManagerProtocolClientSideTranslatorPB.java | 56 +- .../ozone/snapshot/CancelSnapshotDiffResponse.java | 2 +- .../ozone/snapshot/ListSnapshotResponse.java | 37 +- .../ozone/snapshot/SnapshotDiffReportOzone.java | 8 + .../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 2 + hadoop-ozone/csi/pom.xml | 35 + .../org/apache/hadoop/ozone/csi/CsiServer.java | 4 +- hadoop-ozone/datanode/pom.xml | 7 + hadoop-ozone/dev-support/checks/acceptance.sh | 4 +- hadoop-ozone/dev-support/checks/blockade.sh | 2 +- hadoop-ozone/dev-support/checks/build.sh | 2 +- hadoop-ozone/dev-support/checks/coverage.sh | 4 +- hadoop-ozone/dev-support/checks/junit.sh | 8 +- hadoop-ozone/dev-support/checks/kubernetes.sh | 2 +- hadoop-ozone/dev-support/checks/native.sh | 2 +- hadoop-ozone/dev-support/checks/unit.sh | 1 - .../dist/dev-support/bin/dist-layout-stitching | 3 + hadoop-ozone/dist/pom.xml | 9 +- .../dist/src/main/compose/common/ec-test.sh | 2 + .../dashboards/Ozone - CreateKey Metrics.json | 3730 ++++ .../dashboards/Ozone - DeleteKey Metrics.json | 1774 ++ .../dashboards/Ozone - ReadKey Metrics.json | 3221 ++++ .../src/main/compose/common/hadoop-secure.yaml | 4 +- .../dist/src/main/compose/common/hadoop.yaml | 1 - .../dist/src/main/compose/common/s3-haproxy.yaml | 2 - .../dist/src/main/compose/common/s3a-test.sh | 3 +- .../main/compose/compatibility/docker-compose.yaml | 2 - .../src/main/compose/compatibility/docker-config | 1 + .../compose/ozone-balancer/docker-compose.yaml | 2 - .../src/main/compose/ozone-balancer/docker-config | 1 + .../src/main/compose/ozone-csi/docker-compose.yaml | 2 - .../dist/src/main/compose/ozone-csi/docker-config | 1 + .../src/main/compose/ozone-ha/docker-compose.yaml | 2 - .../dist/src/main/compose/ozone-ha/docker-config | 1 + .../dist/src/main/compose/ozone-om-ha/Dockerfile | 5 +- .../main/compose/ozone-om-ha/docker-compose.yaml | 1 - .../src/main/compose/ozone-om-ha/docker-config | 1 + .../compose/ozone-om-prepare/docker-compose.yaml | 2 - .../main/compose/ozone-om-prepare/docker-config | 1 + .../compose/ozone-topology/docker-compose.yaml | 1 - .../src/main/compose/ozone-topology/docker-config | 1 + .../src/main/compose/ozone/docker-compose.yaml | 2 - .../dist/src/main/compose/ozone/docker-config | 1 + .../dist/src/main/compose/ozone/freon-ockg.yaml | 1 - .../dist/src/main/compose/ozone/freon-rk.yaml | 1 - .../dist/src/main/compose/ozone/legacy-bucket.yaml | 2 - .../dist/src/main/compose/ozone/monitoring.yaml | 2 - .../dist/src/main/compose/ozone/profiling.yaml | 2 - .../main/compose/ozoneblockade/docker-compose.yaml | 1 - .../src/main/compose/ozoneblockade/docker-config | 1 + .../dist/src/main/compose/ozonescripts/Dockerfile | 7 +- .../main/compose/ozonescripts/docker-compose.yaml | 1 - .../src/main/compose/ozonescripts/docker-config | 1 + .../compose/ozonesecure-ha/docker-compose.yaml | 17 +- .../src/main/compose/ozonesecure-ha/docker-config | 3 +- .../main/compose/ozonesecure-ha/om-bootstrap.yaml | 2 - .../compose/ozonesecure-ha/root-ca-rotation.yaml | 2 - .../compose/ozonesecure-ha/s3g-virtual-host.yaml | 2 - .../compose/ozonesecure-ha/scm-decommission.yaml | 1 - .../compose/ozonesecure-mr/docker-compose.yaml | 7 +- .../src/main/compose/ozonesecure-mr/docker-config | 1 + .../compose/ozonesecure/certificate-rotation.yaml | 2 - .../main/compose/ozonesecure/docker-compose.yaml | 11 +- .../src/main/compose/ozonesecure/docker-config | 1 + .../dist/src/main/compose/ozonesecure/fcq.yaml | 2 - .../main/compose/ozonesecure/root-ca-rotation.yaml | 2 - .../dist/src/main/compose/ozonesecure/vault.yaml | 1 - .../src/main/compose/restart/docker-compose.yaml | 2 - .../dist/src/main/compose/restart/docker-config | 1 + hadoop-ozone/dist/src/main/compose/test-all.sh | 2 +- hadoop-ozone/dist/src/main/compose/testlib.sh | 11 +- .../compose/upgrade/compose/ha/docker-compose.yaml | 3 +- .../main/compose/upgrade/compose/ha/docker-config | 1 + .../upgrade/compose/non-ha/docker-compose.yaml | 2 - .../compose/upgrade/compose/non-ha/docker-config | 1 + .../upgrade/compose/om-ha/docker-compose.yaml | 2 - .../compose/upgrade/compose/om-ha/docker-config | 1 + .../dist/src/main/compose/xcompat/clients.yaml | 2 - .../dist/src/main/compose/xcompat/docker-config | 1 + .../dist/src/main/compose/xcompat/new-cluster.yaml | 2 - .../dist/src/main/compose/xcompat/old-cluster.yaml | 2 - hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 2 +- .../dist/src/main/keytabs/update-keytabs.sh | 2 +- hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 14 +- hadoop-ozone/dist/src/main/license/jar-report.txt | 14 +- .../dist/src/main/license/update-jar-report.sh | 2 +- .../dist/src/main/smoketest/admincli/cert.robot | 13 +- .../src/main/smoketest/admincli/datanode.robot | 33 +- .../src/main/smoketest/balancer/testBalancer.robot | 40 + .../src/main/smoketest/basic/ozone-shell-lib.robot | 27 +- .../dist/src/main/smoketest/cli/classpath.robot | 30 +- .../debug/ozone-debug-corrupt-block.robot | 2 +- .../debug/ozone-debug-dead-datanode.robot | 2 +- .../debug/ozone-debug-lease-recovery.robot | 4 +- .../debug/ozone-debug-stale-datanode.robot | 2 +- .../main/smoketest/debug/ozone-debug-tests.robot | 6 +- .../src/main/smoketest/debug/ozone-debug.robot | 6 +- .../dist/src/main/smoketest/ec/rewrite.robot | 45 + .../src/main/smoketest/httpfs/operations.robot | 5 - .../main/smoketest/httpfs/operations_tests.robot | 7 +- hadoop-ozone/dist/src/main/smoketest/lib/os.robot | 23 +- .../omha/data-creation-before-om-bootstrap.robot | 22 +- .../dist/src/main/smoketest/ozone-lib/shell.robot | 12 +- .../src/main/smoketest/ozone-lib/shell_tests.robot | 2 +- .../src/main/smoketest/s3/MultipartUpload.robot | 62 +- .../dist/src/main/smoketest/s3/boto_client.py | 21 +- .../dist/src/main/smoketest/s3/commonawslib.robot | 9 +- .../dist/src/main/smoketest/s3/objectcopy.robot | 4 +- .../dist/src/main/smoketest/s3/objectputget.robot | 6 +- .../src/main/smoketest/s3/secretgenerate.robot | 4 +- .../dist/src/main/smoketest/s3/secretrevoke.robot | 4 +- .../src/main/smoketest/security/admin-cert.robot | 5 +- .../smoketest/security/ozone-secure-token.robot | 21 +- .../main/smoketest/snapshot/snapshot-acls.robot | 6 +- .../src/main/smoketest/snapshot/snapshot-sh.robot | 10 + .../dist/src/main/smoketest/spnego/web.robot | 83 +- .../dist/src/shell/ozone/ozone-functions.sh | 5 +- .../fault-injection-test/mini-chaos-tests/pom.xml | 7 + .../fault-injection-test/network-tests/pom.xml | 7 + hadoop-ozone/httpfsgateway/pom.xml | 34 +- .../fs/http/server/HttpFSExceptionProvider.java | 2 +- .../fs/http/server/HttpFSParametersProvider.java | 3 - .../apache/ozone/fs/http/server/HttpFSServer.java | 34 +- .../fs/http/server/HttpFSServerWebServer.java | 10 +- .../apache/ozone/lib/wsrs/ParametersProvider.java | 50 +- .../main/resources/webapps/webhdfs/WEB-INF/web.xml | 16 +- .../httpfsgateway/src/main/webapp/WEB-INF/web.xml | 16 +- hadoop-ozone/insight/pom.xml | 37 + .../dev-support/findbugsExcludeFile.xml | 14 +- hadoop-ozone/integration-test/pom.xml | 7 + .../fs/ozone/AbstractOzoneFileSystemTest.java | 15 + .../ozone/AbstractRootedOzoneFileSystemTest.java | 6 + .../TestSCMContainerPlacementPolicyMetrics.java | 163 - .../apache/hadoop/hdds/scm/TestSecretKeysApi.java | 2 + .../hadoop/hdds/scm/TestXceiverClientGrpc.java | 22 +- .../TestContainerStateManagerIntegration.java | 5 +- .../hdds/scm/pipeline/TestPipelineClose.java | 14 +- .../hdds/scm/storage/TestContainerCommandsEC.java | 10 +- .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 1 - .../hadoop/ozone/TestContainerOperations.java | 56 + .../TestContainerStateMachineIdempotency.java | 116 - .../apache/hadoop/ozone/TestDelegationToken.java | 2 - .../hadoop/ozone/TestMultipartObjectGet.java | 10 + .../hadoop/ozone/TestOzoneConfigurationFields.java | 1 - .../hadoop/ozone/TestSecureOzoneCluster.java | 157 +- ...lientAbstract.java => OzoneRpcClientTests.java} | 659 +- .../ozone/client/rpc/TestBlockOutputStream.java | 2 +- ...ayCommitInRatis.java => TestCommitInRatis.java} | 30 +- .../client/rpc/TestFailureHandlingByClient.java | 19 +- .../client/rpc/TestOzoneAtRestEncryption.java | 8 - .../rpc/TestOzoneClientRetriesOnExceptions.java | 4 +- .../ozone/client/rpc/TestOzoneRpcClient.java | 16 +- .../client/rpc/TestOzoneRpcClientWithRatis.java | 347 +- .../ozone/client/rpc/TestSecureOzoneRpcClient.java | 125 +- .../ozone/client/rpc/TestWatchForCommit.java | 33 +- .../ozone/container/TestContainerReplication.java | 211 + .../container/metrics/TestContainerMetrics.java | 193 +- .../org/apache/hadoop/ozone/debug/TestLDBCli.java | 51 + .../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 7 +- .../hadoop/ozone/om/TestOMUpgradeFinalization.java | 19 + .../hadoop/ozone/om/TestSecureOzoneManager.java | 4 +- .../om/snapshot/TestSnapshotDeletingService.java | 3 + .../TestSnapshotDirectoryCleaningService.java | 2 + .../ozone/recon/TestReconAndAdminContainerCLI.java | 2 + .../hadoop/ozone/recon/TestReconScmHASnapshot.java | 2 +- ...napshot.java => TestReconScmNonHASnapshot.java} | 11 +- .../hadoop/ozone/shell/TestNSSummaryAdmin.java | 20 +- .../shell/TestOzoneContainerUpgradeShell.java | 2 + .../hadoop/ozone/shell/TestOzoneDebugShell.java | 32 +- .../hadoop/ozone/shell/TestOzoneRepairShell.java | 133 + .../hadoop/ozone/shell/TestReconfigShell.java | 3 + hadoop-ozone/interface-client/pom.xml | 7 + .../src/main/proto/OmClientProtocol.proto | 23 + hadoop-ozone/interface-storage/pom.xml | 37 + .../apache/hadoop/ozone/om/OMMetadataManager.java | 15 +- hadoop-ozone/ozone-manager/pom.xml | 42 + .../org/apache/hadoop/ozone/audit/OMAction.java | 5 +- .../hadoop/ozone/om/OMPerformanceMetrics.java | 57 + .../hadoop/ozone/om/OmMetadataManagerImpl.java | 99 +- .../apache/hadoop/ozone/om/OmSnapshotManager.java | 63 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 16 +- .../hadoop/ozone/om/OzoneManagerStarter.java | 4 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 6 + .../ozone/om/ratis/OzoneManagerStateMachine.java | 7 + .../hadoop/ozone/om/request/RequestAuditor.java | 4 + .../ozone/om/request/key/OMKeyCommitRequest.java | 24 + .../om/request/key/OMKeyCommitRequestWithFSO.java | 8 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 14 + .../om/request/key/OMKeyCreateRequestWithFSO.java | 1 + .../ozone/om/request/key/OMKeyDeleteRequest.java | 15 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 5 +- .../ozone/om/request/key/OMKeysDeleteRequest.java | 35 +- .../om/request/key/OmKeysDeleteRequestWithFSO.java | 12 +- .../om/request/upgrade/OMCancelPrepareRequest.java | 10 + .../request/upgrade/OMFinalizeUpgradeRequest.java | 10 + .../ozone/om/request/upgrade/OMPrepareRequest.java | 10 + .../om/request/validation/RequestValidations.java | 4 +- .../om/request/validation/ValidatorRegistry.java | 22 +- .../snapshot/OMSnapshotCreateResponse.java | 2 +- .../ozone/om/service/DirectoryDeletingService.java | 11 - .../ozone/om/service/KeyDeletingService.java | 10 - .../ozone/om/service/SnapshotDeletingService.java | 14 +- .../protocolPB/OzoneManagerRequestHandler.java | 18 +- .../apache/hadoop/ozone/om/TestChunkStreams.java | 5 +- .../hadoop/ozone/om/TestOmMetadataManager.java | 61 +- .../hadoop/ozone/om/TestOmSnapshotManager.java | 17 +- .../om/request/key/TestOMKeyCommitRequest.java | 88 +- .../request/key/TestOMKeyCommitRequestWithFSO.java | 24 +- .../om/request/key/TestOMKeyCreateRequest.java | 134 +- .../request/key/TestOMKeyCreateRequestWithFSO.java | 8 +- .../ozone/om/request/key/TestOMKeyRequest.java | 2 + .../om/request/key/TestOMKeysDeleteRequest.java | 7 + .../snapshot/TestOMSnapshotCreateRequest.java | 4 +- .../snapshot/TestOMSnapshotDeleteRequest.java | 4 +- .../snapshot/TestOMSnapshotRenameRequest.java | 4 +- .../om/service/TestSnapshotDeletingService.java | 100 + .../ozone/om/snapshot/TestOmSnapshotUtils.java | 44 - .../ozone/om/snapshot/TestSnapshotDiffManager.java | 2 + .../security/TestOmCertificateClientInit.java | 4 +- .../TestOzoneDelegationTokenSecretManager.java | 8 +- hadoop-ozone/ozonefs-common/pom.xml | 8 +- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 6 + .../ozone/BasicRootedOzoneClientAdapterImpl.java | 3 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 6 + hadoop-ozone/ozonefs-hadoop2/pom.xml | 11 + hadoop-ozone/ozonefs-hadoop3-client/pom.xml | 7 + hadoop-ozone/ozonefs-hadoop3/pom.xml | 7 + hadoop-ozone/ozonefs-shaded/pom.xml | 7 + hadoop-ozone/ozonefs/pom.xml | 7 + hadoop-ozone/recon-codegen/pom.xml | 42 + hadoop-ozone/recon/pom.xml | 55 +- .../apache/hadoop/ozone/recon/ReconConstants.java | 4 + .../apache/hadoop/ozone/recon/ReconContext.java | 5 +- .../hadoop/ozone/recon/ReconResponseUtils.java | 84 + .../org/apache/hadoop/ozone/recon/ReconServer.java | 19 +- .../org/apache/hadoop/ozone/recon/ReconUtils.java | 185 +- .../ozone/recon/api/ClusterStateEndpoint.java | 13 +- .../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 25 +- .../hadoop/ozone/recon/api/NodeEndpoint.java | 10 +- .../ozone/recon/api/OMDBInsightEndpoint.java | 612 +- .../ozone/recon/api/OMDBInsightSearchEndpoint.java | 389 + .../ozone/recon/api/handlers/EntityHandler.java | 2 +- .../ozone/recon/api/handlers/FSOBucketHandler.java | 1 - .../recon/api/types/ClusterStateResponse.java | 29 + .../ozone/recon/api/types/KeyEntityInfo.java | 42 +- .../ozone/recon/api/types/ListKeysResponse.java | 114 + .../hadoop/ozone/recon/api/types/ParamInfo.java | 133 + .../ozone/recon/fsck/ContainerHealthTask.java | 14 +- .../recon/recovery/ReconOmMetadataManagerImpl.java | 6 - .../hadoop/ozone/recon/scm/PipelineSyncTask.java | 2 +- .../ozone/recon/scm/ReconContainerManager.java | 8 +- .../ozone/recon/scm/ReconPipelineManager.java | 79 +- .../recon/scm/ReconPipelineReportHandler.java | 7 +- .../scm/ReconStorageContainerManagerFacade.java | 4 + .../spi/impl/OzoneManagerServiceProviderImpl.java | 125 +- .../impl/StorageContainerServiceProviderImpl.java | 110 +- .../ozone/recon/tasks/ContainerKeyMapperTask.java | 8 +- .../ozone/recon/tasks/ContainerSizeCountTask.java | 6 +- .../ozone/recon/tasks/FileSizeCountTask.java | 6 +- .../hadoop/ozone/recon/tasks/NSSummaryTask.java | 2 +- .../ozone/recon/tasks/NSSummaryTaskWithFSO.java | 4 +- .../ozone/recon/tasks/NSSummaryTaskWithLegacy.java | 16 +- .../ozone/recon/tasks/NSSummaryTaskWithOBS.java | 6 +- .../ozone/recon/tasks/OMDBUpdatesHandler.java | 11 +- .../ozone/recon/tasks/OmTableInsightTask.java | 4 +- .../webapps/recon/ozone-recon-web/.eslintignore | 16 +- .../webapps/recon/ozone-recon-web/.eslintrc.json | 112 + .../webapps/recon/ozone-recon-web/.prettierrc.js | 28 + .../recon/ozone-recon-web/.vscode/extensions.json | 6 + .../recon/ozone-recon-web/.vscode/settings.json | 9 + .../webapps/recon/ozone-recon-web/api/db.json | 451 +- .../webapps/recon/ozone-recon-web/api/routes.json | 31 +- .../recon/ozone-recon-web/config-overrides.js | 33 - .../webapps/recon/ozone-recon-web/index.html | 55 + .../webapps/recon/ozone-recon-web/package.json | 159 +- .../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 19224 +++++-------------- .../recon/ozone-recon-web/public/index.html | 45 - .../webapps/recon/ozone-recon-web/src/app.less | 28 +- .../webapps/recon/ozone-recon-web/src/app.tsx | 50 +- .../src/components/aclDrawer/aclDrawer.tsx | 29 +- .../components/autoReloadPanel/autoReloadPanel.tsx | 59 +- .../src/components/breadcrumbs/breadcrumbs.tsx | 18 +- .../src/components/eChart/eChart.tsx | 87 + .../src/components/multiSelect/multiSelect.tsx | 14 +- .../src/components/navBar/navBar.tsx | 176 +- .../src/components/overviewCard/overviewCard.less | 9 +- .../src/components/overviewCard/overviewCard.tsx | 87 +- .../src/components/quotaBar/quotaBar.tsx | 16 +- .../src/components/rightDrawer/rightDrawer.tsx | 31 +- .../src/components/storageBar/storageBar.less | 3 +- .../src/components/storageBar/storageBar.tsx | 35 +- .../src/constants/breadcrumbs.constants.tsx | 4 +- .../webapps/recon/ozone-recon-web/src/routes.tsx | 24 +- .../ozone-recon-web/src/utils/autoReloadHelper.tsx | 4 +- .../src/utils/axiosRequestHelper.tsx | 28 +- .../ozone-recon-web/src/utils/columnSearch.tsx | 16 +- .../recon/ozone-recon-web/src/utils/common.tsx | 10 +- .../recon/ozone-recon-web/src/utils/themeIcons.tsx | 18 +- .../ozone-recon-web/src/views/buckets/buckets.tsx | 154 +- .../src/views/datanodes/datanodes.less | 4 + .../src/views/datanodes/datanodes.tsx | 221 +- .../src/views/diskUsage/diskUsage.less | 34 +- .../src/views/diskUsage/diskUsage.tsx | 267 +- .../views/heatMap/constants/heatmapConstants.tsx | 2 +- .../src/views/heatMap/heatMapConfiguration.tsx | 10 +- .../ozone-recon-web/src/views/heatMap/heatmap.less | 5 +- .../ozone-recon-web/src/views/heatMap/heatmap.tsx | 228 +- .../src/views/insights/insights.tsx | 426 +- .../ozone-recon-web/src/views/insights/om/om.tsx | 340 +- .../views/missingContainers/missingContainers.tsx | 111 +- .../src/views/overview/overview.tsx | 323 +- .../src/views/pipelines/pipelines.tsx | 77 +- .../ozone-recon-web/src/views/volumes/volumes.tsx | 96 +- .../webapps/recon/ozone-recon-web/tsconfig.json | 12 +- .../webapps/recon/ozone-recon-web/vite-env.d.ts | 18 + .../webapps/recon/ozone-recon-web/vite.config.ts | 83 + .../ozone/recon/OMMetadataManagerTestUtils.java | 57 +- ...teCounts.java => TestClusterStateEndpoint.java} | 28 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 11 +- .../recon/api/TestOMDBInsightSearchEndpoint.java | 796 + .../ozone/recon/api/TestOmDBInsightEndPoint.java | 1116 +- .../ozone/recon/api/TestTriggerDBSyncEndpoint.java | 3 +- .../ozone/recon/scm/TestReconPipelineManager.java | 36 +- .../impl/TestOzoneManagerServiceProviderImpl.java | 101 +- .../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 67 + hadoop-ozone/s3-secret-store/pom.xml | 12 + hadoop-ozone/s3gateway/pom.xml | 55 +- .../java/org/apache/hadoop/ozone/s3/Gateway.java | 3 +- .../hadoop/ozone/s3/RootPageDisplayFilter.java | 7 +- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 61 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 27 +- .../org/apache/hadoop/ozone/s3/util/S3Consts.java | 2 + .../ozone/s3secret/S3SecretManagementEndpoint.java | 7 +- .../hadoop/ozone/client/ClientProtocolStub.java | 20 +- .../hadoop/ozone/client/OzoneBucketStub.java | 44 + .../ozone/s3/endpoint/TestPermissionCheck.java | 9 +- .../hadoop/ozone/s3secret/TestSecretGenerate.java | 2 + .../hadoop/ozone/s3secret/TestSecretRevoke.java | 2 + hadoop-ozone/tools/pom.xml | 42 + .../ozone/admin/nssummary/DiskUsageSubCommand.java | 4 - .../ozone/admin/nssummary/NSSummaryAdmin.java | 6 + .../ozone/admin/om/DecommissionOMSubcommand.java | 4 +- .../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 2 +- .../org/apache/hadoop/ozone/debug/DBScanner.java | 26 +- .../apache/hadoop/ozone/debug/RocksDBUtils.java | 31 + .../org/apache/hadoop/ozone/debug/ValueSchema.java | 185 + .../ozone/freon/FreonReplicationOptions.java | 9 +- .../hadoop/ozone/repair/TransactionInfoRepair.java | 135 + .../hadoop/ozone/repair/om/SnapshotRepair.java | 34 +- .../ozone/shell/MandatoryReplicationOptions.java | 32 +- .../hadoop/ozone/shell/ReplicationOptions.java | 19 +- .../ozone/shell/ShellReplicationOptions.java | 10 +- .../hadoop/ozone/shell/keys/KeyCommands.java | 1 + .../hadoop/ozone/shell/keys/PutKeyHandler.java | 26 +- .../hadoop/ozone/shell/keys/RewriteKeyHandler.java | 71 + .../ozone/shell/snapshot/ListSnapshotHandler.java | 2 +- .../ozone/shell/snapshot/SnapshotDiffHandler.java | 84 +- .../ozone/repair/TestTransactionInfoRepair.java | 146 + .../org.mockito.plugins.MockMaker} | 14 +- pom.xml | 201 +- 615 files changed, 28780 insertions(+), 25399 deletions(-) diff --cc hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java index 745790abc8,c058cb4d44..2099541dac --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java @@@ -103,8 -103,9 +103,10 @@@ import org.apache.hadoop.hdds.protocol. import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartContainerBalancerResponseProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StopContainerBalancerRequestProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ResetDeletedBlockRetryCountRequestProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ReconcileContainerRequestProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.Type; + import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfoResponseProto; + import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfoRequestProto; import org.apache.hadoop.hdds.scm.DatanodeAdminError; import org.apache.hadoop.hdds.scm.ScmInfo; import org.apache.hadoop.hdds.scm.container.ContainerID; diff --cc hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto index ea63b82c8c,039914369b..dd4350ae49 --- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto +++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto @@@ -84,7 -84,7 +84,8 @@@ message ScmContainerLocationRequest optional SingleNodeQueryRequestProto singleNodeQueryRequest = 45; optional GetContainersOnDecomNodeRequestProto getContainersOnDecomNodeRequest = 46; optional GetMetricsRequestProto getMetricsRequest = 47; - optional ReconcileContainerRequestProto reconcileContainerRequest = 48; + optional ContainerBalancerStatusInfoRequestProto containerBalancerStatusInfoRequest = 48; ++ optional ReconcileContainerRequestProto reconcileContainerRequest = 49; } message ScmContainerLocationResponse { @@@ -140,7 -140,7 +141,8 @@@ optional SingleNodeQueryResponseProto singleNodeQueryResponse = 45; optional GetContainersOnDecomNodeResponseProto getContainersOnDecomNodeResponse = 46; optional GetMetricsResponseProto getMetricsResponse = 47; - optional ReconcileContainerResponseProto reconcileContainerResponse = 48; + optional ContainerBalancerStatusInfoResponseProto containerBalancerStatusInfoResponse = 48; ++ optional ReconcileContainerResponseProto reconcileContainerResponse = 49; enum Status { OK = 1; @@@ -195,7 -195,7 +197,8 @@@ enum Type SingleNodeQuery = 41; GetContainersOnDecomNode = 42; GetMetrics = 43; - ReconcileContainer = 44; + GetContainerBalancerStatusInfo = 44; ++ ReconcileContainer = 45; } /** diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java index 3b2a84f4f3,4fcf130117..2b2c032cff --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java @@@ -208,22 -207,6 +207,13 @@@ public final class SCMEvents new TypedEvent<>(CommandStatusReportHandler.DeleteBlockStatus.class, "Delete_Block_Status"); - /** - * A CRL status report will be sent by datanodes. This report is received - * and processed by SCMDatanodeHeartbeatDispatcher. - */ - public static final TypedEvent<CRLStatusReportFromDatanode> - CRL_STATUS_REPORT = - new TypedEvent<>(CRLStatusReportFromDatanode.class, - "Crl_Status_Report"); - + /** + * This event will be triggered whenever a datanode needs to reconcile its replica of a container with other + * replicas in the cluster. + */ + public static final TypedEvent<ContainerID> + RECONCILE_CONTAINER = new TypedEvent<>(ContainerID.class, "Reconcile_Container"); + /** * Private Ctor. Never Constructed. */ diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 9cf44677eb,828b452d30..4bba6b7ac2 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@@ -55,8 -59,7 +59,9 @@@ import org.apache.hadoop.hdds.scm.conta import org.apache.hadoop.hdds.scm.container.balancer.IllegalContainerBalancerStateException; import org.apache.hadoop.hdds.scm.container.balancer.InvalidContainerBalancerConfigurationException; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; +import org.apache.hadoop.hdds.scm.container.reconciliation.ReconciliationEligibilityHandler; +import org.apache.hadoop.hdds.scm.container.reconciliation.ReconciliationEligibilityHandler.EligibilityResult; + import org.apache.hadoop.hdds.scm.container.common.helpers.DeletedBlocksTransactionInfoWrapper; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes; diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 33fbf5fb76,5c0248f162..7ee17d5837 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@@ -49,8 -49,8 +49,9 @@@ import org.apache.hadoop.hdds.scm.ScmUt import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerManagerImpl; import org.apache.hadoop.hdds.scm.PlacementPolicyValidateProxy; +import org.apache.hadoop.hdds.scm.container.reconciliation.ReconcileContainerEventHandler; import org.apache.hadoop.hdds.scm.container.balancer.MoveManager; + import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMPerformanceMetrics; import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaPendingOps; import org.apache.hadoop.hdds.scm.container.replication.DatanodeCommandCountUpdatedHandler; import org.apache.hadoop.hdds.scm.container.replication.LegacyReplicationManager; @@@ -504,12 -506,7 +507,10 @@@ public final class StorageContainerMana scmNodeManager, containerManager, scmContext); PipelineActionHandler pipelineActionHandler = new PipelineActionHandler(pipelineManager, scmContext, configuration); - CRLStatusReportHandler crlStatusReportHandler = - new CRLStatusReportHandler(certificateStore, configuration); + ReconcileContainerEventHandler reconcileContainerEventHandler = + new ReconcileContainerEventHandler(containerManager, scmContext); + eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager); eventQueue.addHandler(SCMEvents.RETRIABLE_DATANODE_COMMAND, scmNodeManager); eventQueue.addHandler(SCMEvents.NODE_REPORT, nodeReportHandler); @@@ -581,8 -578,6 +582,7 @@@ (DeletedBlockLogImpl) scmBlockManager.getDeletedBlockLog()); eventQueue.addHandler(SCMEvents.PIPELINE_ACTIONS, pipelineActionHandler); eventQueue.addHandler(SCMEvents.PIPELINE_REPORT, pipelineReportHandler); - eventQueue.addHandler(SCMEvents.CRL_STATUS_REPORT, crlStatusReportHandler); + eventQueue.addHandler(SCMEvents.RECONCILE_CONTAINER, reconcileContainerEventHandler); scmNodeManager.registerSendCommandNotify( SCMCommandProto.Type.deleteBlocksCommand, --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
