This is an automated email from the ASF dual-hosted git repository. erose pushed a commit to branch HDDS-10239-container-reconciliation in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 3355701853c10f47c0acafbef502a3d73e48926a Merge: 0baba47daa fb43023f79 Author: Ethan Rose <[email protected]> AuthorDate: Wed Aug 21 15:37:15 2024 -0400 Merge branch 'master' into HDDS-10239-container-reconciliation * master: (50 commits) HDDS-11331. Fix Datanode unable to report for a long time (#7090) HDDS-11346. FS CLI gives incorrect recursive volume deletion prompt (#7102) HDDS-11349. Add NullPointer handling when volume/bucket tables are not initialized (#7103) HDDS-11209. Avoid insufficient EC pipelines in the container pipeline cache (#6974) HDDS-11284. refactor quota repair non-blocking while upgrade (#7035) HDDS-9790. Add tests for Overview page (#6983) HDDS-10904. [hsync] Enable PutBlock piggybacking and incremental chunk list by default (#7074) HDDS-11322. [hsync] Block ECKeyOutputStream from calling hsync and hflush (#7098) HDDS-11325. Intermittent failure in TestBlockOutputStreamWithFailures#testContainerClose (#7099) HDDS-11340. Avoid extra PubBlock call when a full block is closed (#7094) HDDS-11155. Improve Volumes page UI (#7048) HDDS-11324. Negative value preOpLatencyMs in DN audit log (#7093) HDDS-11246. [Recon] Use optional chaining instead of explicit undefined check for Objects in Container and Pipeline Module. (#7037) HDDS-11323. Mark TestLeaseRecovery as flaky HDDS-11338. Bump zstd-jni to 1.5.6-4 (#7085) HDDS-11337. Bump Spring Framework to 5.3.39 (#7084) HDDS-11327. [hsync] Revert config default ozone.fs.hsync.enabled to false (#7079) HDDS-11325. Mark testWriteMoreThanMaxFlushSize as flaky HDDS-11336. Bump slf4j to 2.0.16 (#7086) HDDS-11335. Bump exec-maven-plugin to 3.4.1 (#7087) ... Conflicts: hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java hadoop-hdds/client/pom.xml | 57 ++ .../apache/hadoop/hdds/scm/OzoneClientConfig.java | 24 +- .../hadoop/hdds/scm/XceiverClientCreator.java | 117 +++ .../hadoop/hdds/scm/XceiverClientFactory.java | 41 +- .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 9 - .../hadoop/hdds/scm/XceiverClientManager.java | 117 +-- .../apache/hadoop/hdds/scm/XceiverClientRatis.java | 55 +- .../hadoop/hdds/scm/client/HddsClientUtils.java | 3 +- .../hdds/scm/storage/AbstractCommitWatcher.java | 17 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 32 +- .../hdds/scm/storage/ECBlockOutputStream.java | 2 +- .../hdds/scm/storage/RatisBlockOutputStream.java | 3 +- .../storage/TestBlockOutputStreamCorrectness.java | 5 +- hadoop-hdds/common/pom.xml | 52 ++ .../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 + .../apache/hadoop/hdds/conf/RatisConfUtils.java | 44 + .../hadoop/hdds/protocol/DatanodeDetails.java | 3 + .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 12 + .../apache/hadoop/hdds/scm/XceiverClientSpi.java | 8 +- .../x509/certificate/client/CertificateClient.java | 11 +- .../certificate/utils/CertificateSignRequest.java | 59 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 2 +- .../common/src/main/resources/ozone-default.xml | 52 +- .../hadoop/hdds/conf/TestRatisConfUtils.java | 57 ++ .../hadoop/hdds/protocol/TestDatanodeDetails.java | 19 + hadoop-hdds/container-service/pom.xml | 116 +++ .../ozone/HddsDatanodeClientProtocolServer.java | 9 +- .../apache/hadoop/ozone/HddsDatanodeService.java | 6 +- .../container/common/impl/HddsDispatcher.java | 65 +- .../common/statemachine/DatanodeConfiguration.java | 43 + .../common/statemachine/DatanodeStateMachine.java | 9 +- .../common/statemachine/EndpointStateMachine.java | 72 +- .../common/statemachine/StateContext.java | 164 ++-- .../server/ratis/ContainerStateMachine.java | 53 +- .../transport/server/ratis/XceiverServerRatis.java | 33 +- .../ozone/container/common/volume/HddsVolume.java | 26 +- .../container/common/volume/VolumeInfoMetrics.java | 8 + .../container/keyvalue/KeyValueContainerCheck.java | 11 +- .../KeyValueContainerMetadataInspector.java | 17 +- .../container/keyvalue/helpers/ChunkUtils.java | 95 +- .../container/keyvalue/impl/BlockManagerImpl.java | 2 +- .../statemachine/background/BlockDeletingTask.java | 26 +- .../ozone/container/metadata/DatanodeStore.java | 5 + .../metadata/DatanodeStoreSchemaThreeImpl.java | 13 + .../DatanodeStoreWithIncrementalChunkList.java | 23 +- .../ozone/container/ozoneimpl/OzoneContainer.java | 43 +- ...inerDatanodeProtocolClientSideTranslatorPB.java | 4 +- .../ozone/container/common/ContainerTestUtils.java | 2 +- .../hadoop/ozone/container/common/ScmTestMock.java | 8 +- .../container/keyvalue/TestKeyValueContainer.java | 3 +- .../container/keyvalue/helpers/TestChunkUtils.java | 108 ++- .../upgrade/TestDatanodeUpgradeToHBaseSupport.java | 242 +---- .../upgrade/TestDatanodeUpgradeToSchemaV3.java | 382 ++------ .../upgrade/TestDatanodeUpgradeToScmHA.java | 331 ++----- .../ozone/container/upgrade/UpgradeTestHelper.java | 271 ++++++ hadoop-hdds/docs/content/interface/ReconApi.md | 29 +- hadoop-hdds/erasurecode/pom.xml | 15 + hadoop-hdds/framework/pom.xml | 126 +++ .../certificate/client/DNCertificateClient.java | 15 +- .../client/DefaultCertificateClient.java | 42 +- .../certificate/client/SCMCertificateClient.java | 33 +- .../org/apache/hadoop/hdds/utils/db/DBStore.java | 8 + .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 8 + .../org/apache/hadoop/ozone/audit/AuditLogger.java | 10 +- .../apache/hadoop/ozone/audit/AuditLoggerType.java | 3 +- .../certificate/authority/TestDefaultCAServer.java | 20 +- .../certificate/authority/TestDefaultProfile.java | 44 +- .../client/CertificateClientTestImpl.java | 19 +- .../client/TestDefaultCertificateClient.java | 15 +- .../utils/TestCertificateSignRequest.java | 31 +- hadoop-hdds/interface-server/pom.xml | 4 + hadoop-hdds/managed-rocksdb/pom.xml | 20 + hadoop-hdds/rocks-native/pom.xml | 23 + hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 31 + hadoop-hdds/server-scm/pom.xml | 110 +++ .../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 14 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 78 +- .../hdds/scm/security/RootCARotationManager.java | 2 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 7 +- .../hdds/scm/server/SCMClientProtocolServer.java | 7 +- .../hdds/scm/server/SCMDatanodeProtocolServer.java | 7 +- .../hdds/scm/server/SCMSecurityProtocolServer.java | 5 +- .../hdds/scm/server/StorageContainerManager.java | 4 +- .../main/resources/webapps/scm/scm-overview.html | 26 + .../src/main/resources/webapps/scm/scm.js | 14 + hadoop-hdds/tools/pom.xml | 61 ++ hadoop-ozone/client/pom.xml | 54 ++ .../ozone/client/io/BlockOutputStreamEntry.java | 2 +- .../hadoop/ozone/client/io/ECKeyOutputStream.java | 11 + .../hadoop/ozone/client/io/KeyOutputStream.java | 111 ++- .../ozone/client/io/KeyOutputStreamSemaphore.java | 71 ++ .../hadoop/ozone/client/MockXceiverClientSpi.java | 5 - .../ozone/client/io/TestKeyOutputStream.java | 126 +++ hadoop-ozone/common/pom.xml | 84 ++ .../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 + hadoop-ozone/csi/pom.xml | 39 + hadoop-ozone/dev-support/checks/acceptance.sh | 5 - hadoop-ozone/dist/src/main/compose/testlib.sh | 59 +- .../src/main/smoketest/basic/ozone-shell-lib.robot | 29 +- .../dist/src/shell/conf/om-audit-log4j2.properties | 30 +- hadoop-ozone/dist/src/shell/ozone/ozone | 1 + hadoop-ozone/httpfsgateway/pom.xml | 28 + hadoop-ozone/insight/pom.xml | 36 + hadoop-ozone/integration-test/pom.xml | 5 + .../java/org/apache/hadoop/fs/ozone/TestHSync.java | 134 +-- .../apache/hadoop/fs/ozone/TestHSyncUpgrade.java | 240 +++++ .../apache/hadoop/fs/ozone/TestLeaseRecovery.java | 2 + .../hadoop/hdds/utils/ClusterContainersUtil.java | 18 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 3 +- .../client/rpc/AbstractTestECKeyOutputStream.java | 18 + .../ozone/client/rpc/TestBlockOutputStream.java | 8 +- .../rpc/TestBlockOutputStreamWithFailures.java | 1 + .../hadoop/ozone/client/rpc/TestCommitInRatis.java | 4 +- .../ozone/client/rpc/TestWatchForCommit.java | 9 +- .../commandhandler/TestDeleteContainerHandler.java | 41 +- .../transport/server/ratis/TestCSMMetrics.java | 2 +- .../container/metrics/TestContainerMetrics.java | 2 +- .../ozoneimpl/TestOzoneContainerWithTLS.java | 2 +- .../ozoneimpl/TestSecureOzoneContainer.java | 2 +- .../container/server/TestContainerServer.java | 2 +- .../server/TestSecureContainerServer.java | 2 +- .../hadoop/ozone/freon/TestDNRPCLoadGenerator.java | 13 +- .../ozone/om/TestOmContainerLocationCache.java | 110 ++- .../hadoop/ozone/shell/TestOzoneShellHA.java | 33 + hadoop-ozone/interface-client/pom.xml | 20 + .../src/main/proto/OmClientProtocol.proto | 21 +- hadoop-ozone/interface-storage/pom.xml | 21 + hadoop-ozone/ozone-manager/pom.xml | 168 +++- .../apache/hadoop/ozone/audit/OMSystemAction.java | 21 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 19 + .../java/org/apache/hadoop/ozone/om/ScmClient.java | 27 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 14 +- .../om/ratis/utils/OzoneManagerRatisUtils.java | 3 + .../request/snapshot/OMSnapshotPurgeRequest.java | 122 +-- .../snapshot/OMSnapshotSetPropertyRequest.java | 32 +- .../om/request/volume/OMQuotaRepairRequest.java | 191 ++++ .../response/snapshot/OMSnapshotPurgeResponse.java | 2 +- .../om/response/volume/OMQuotaRepairResponse.java | 72 ++ .../hadoop/ozone/om/service/QuotaRepairTask.java | 401 ++++++--- .../ozone/om/upgrade/QuotaRepairUpgradeAction.java | 27 +- .../hadoop/ozone/security/OMCertificateClient.java | 15 +- .../org/apache/hadoop/ozone/om/TestScmClient.java | 2 +- .../ozone/om/service/TestKeyDeletingService.java | 2 +- .../ozone/om/service/TestQuotaRepairTask.java | 68 +- hadoop-ozone/ozonefs-common/pom.xml | 51 ++ .../fs/ozone/BasicRootedOzoneFileSystem.java | 2 +- hadoop-ozone/ozonefs/pom.xml | 34 + hadoop-ozone/pom.xml | 20 + hadoop-ozone/recon-codegen/pom.xml | 18 + hadoop-ozone/recon/pom.xml | 150 ++++ .../recon/recovery/ReconOmMetadataManagerImpl.java | 15 +- .../recon/security/ReconCertificateClient.java | 15 +- .../webapps/recon/ozone-recon-web/api/db.json | 5 +- .../webapps/recon/ozone-recon-web/package.json | 4 + .../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 981 ++++++++++++++++++++- .../src/__tests__/Overview.test.tsx | 214 +++++ .../src/__tests__/locators/locators.ts | 48 + .../mocks/overviewMocks/overviewResponseMocks.ts} | 39 +- .../mocks/overviewMocks/overviewServer.ts | 79 ++ .../ozone-recon-web/src/__tests__/vitest.setup.ts | 61 ++ .../webapps/recon/ozone-recon-web/src/app.less | 7 + .../webapps/recon/ozone-recon-web/src/app.tsx | 18 +- .../components/autoReloadPanel/autoReloadPanel.tsx | 19 +- .../src/components/overviewCard/overviewCard.tsx | 4 +- .../recon/ozone-recon-web/src/utils/common.tsx | 5 +- .../src/v2/components/aclDrawer/aclDrawer.tsx | 119 +++ .../src/v2/components/eChart/eChart.tsx | 89 ++ .../components/errorBoundary/errorBoundary.tsx} | 43 +- .../components/loader/loader.tsx} | 29 +- .../overviewCard/overviewCardWrapper.tsx | 79 ++ .../components/overviewCard/overviewSimpleCard.tsx | 148 ++++ .../overviewCard/overviewStorageCard.tsx | 244 +++++ .../overviewCard/overviewSummaryCard.tsx | 111 +++ .../src/v2/components/search/search.tsx | 70 ++ .../src/v2/components/select/columnTag.tsx | 67 ++ .../src/v2/components/select/multiSelect.tsx | 104 +++ .../src/v2/components/select/singleSelect.tsx | 87 ++ .../src/v2/components/storageBar/storageBar.tsx | 93 ++ .../constants/acl.constants.tsx} | 26 +- .../src/v2/constants/select.constants.tsx | 62 ++ .../{app.test.tsx => v2/hooks/debounce.hook.tsx} | 22 +- .../src/v2/pages/overview/overview.less | 26 + .../src/v2/pages/overview/overview.tsx | 542 ++++++++++++ .../src/v2/pages/volumes/volumes.less | 41 + .../src/v2/pages/volumes/volumes.tsx | 353 ++++++++ .../src/{app.test.tsx => v2/routes-v2.tsx} | 21 +- .../src/{app.test.tsx => v2/types/acl.types.ts} | 36 +- .../ozone-recon-web/src/v2/types/bucket.types.ts | 55 ++ .../ozone-recon-web/src/v2/types/overview.types.ts | 66 ++ .../ozone-recon-web/src/v2/types/volume.types.ts | 44 + .../src/views/datanodes/datanodes.tsx | 3 +- .../views/missingContainers/missingContainers.tsx | 8 +- .../src/views/pipelines/pipelines.tsx | 4 +- .../webapps/recon/ozone-recon-web/tsconfig.json | 9 +- .../webapps/recon/ozone-recon-web/vite-env.d.ts | 18 - .../webapps/recon/ozone-recon-web/vite.config.ts | 51 +- hadoop-ozone/s3-secret-store/pom.xml | 8 + hadoop-ozone/s3gateway/pom.xml | 105 +++ hadoop-ozone/tools/pom.xml | 141 +++ .../hadoop/ozone/freon/DNRPCLoadGenerator.java | 7 +- .../hadoop/ozone/freon/DatanodeBlockPutter.java | 7 +- .../hadoop/ozone/freon/DatanodeChunkGenerator.java | 10 +- .../hadoop/ozone/freon/DatanodeChunkValidator.java | 7 +- pom.xml | 114 ++- 204 files changed, 9139 insertions(+), 2135 deletions(-) diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java index 16cacd6fae,22dff7505c..16d282da5a --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java @@@ -113,11 -109,12 +113,16 @@@ public class DatanodeConfiguration exte "hdds.datanode.rocksdb.delete_obsolete_files_period"; public static final Boolean OZONE_DATANODE_CHECK_EMPTY_CONTAINER_DIR_ON_DELETE_DEFAULT = false; + public static final int CONTAINER_CHECKSUM_LOCK_STRIPES_DEFAULT = 127; + public static final int CONTAINER_CLIENT_CACHE_SIZE_DEFAULT = 100; + public static final int + CONTAINER_CLIENT_CACHE_STALE_THRESHOLD_MILLISECONDS_DEFAULT = 10000; + private static final long + AUTO_COMPACTION_SMALL_SST_FILE_INTERVAL_MINUTES_DEFAULT = 120; + private static final int + AUTO_COMPACTION_SMALL_SST_FILE_THREADS_DEFAULT = 1; + /** * Number of threads per volume that Datanode will use for chunk read. */ @@@ -972,15 -934,23 +995,35 @@@ this.autoCompactionSmallSstFileNum = num; } + public int getContainerChecksumLockStripes() { + return containerChecksumLockStripes; + } + + public int getContainerClientCacheSize() { + return containerClientCacheSize; + } + + public int getContainerClientCacheStaleThreshold() { + return containerClientCacheStaleThreshold; + } ++ + public long getAutoCompactionSmallSstFileIntervalMinutes() { + return autoCompactionSmallSstFileIntervalMinutes; + } + + public void setAutoCompactionSmallSstFileIntervalMinutes( + long autoCompactionSmallSstFileIntervalMinutes) { + this.autoCompactionSmallSstFileIntervalMinutes = + autoCompactionSmallSstFileIntervalMinutes; + } + + public int getAutoCompactionSmallSstFileThreads() { + return autoCompactionSmallSstFileThreads; + } + + public void setAutoCompactionSmallSstFileThreads( + int autoCompactionSmallSstFileThreads) { + this.autoCompactionSmallSstFileThreads = + autoCompactionSmallSstFileThreads; + } } diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java index 888d1fd018,a460e30ede..55fcbcdb3c --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java @@@ -44,8 -44,8 +44,9 @@@ import org.apache.hadoop.hdds.security. import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.NettyMetrics; + import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.HddsDatanodeStopService; +import org.apache.hadoop.ozone.container.checksum.DNContainerOperationClient; import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage; import org.apache.hadoop.ozone.container.common.report.ReportManager; import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.CloseContainerCommandHandler; diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index f8034244c1,5cdeaaa578..4523d15991 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@@ -34,7 -34,7 +34,8 @@@ import org.apache.hadoop.hdds.security. import org.apache.hadoop.hdds.security.token.TokenVerifier; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.utils.HddsServerUtil; +import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager; + import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.BlockDeletingService; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; @@@ -122,8 -125,7 +126,8 @@@ public class OzoneContainer private final ReplicationServer replicationServer; private DatanodeDetails datanodeDetails; private StateContext context; + private final ContainerChecksumTreeManager checksumTreeManager; - + private ScheduledExecutorService dbCompactionExecutorService; private final ContainerMetrics metrics; @@@ -239,11 -252,10 +255,11 @@@ blockDeletingServiceTimeout, TimeUnit.MILLISECONDS, blockDeletingServiceWorkerSize, config, datanodeDetails.threadNamePrefix(), + checksumTreeManager, context.getParent().getReconfigurationHandler()); - Duration recoveringContainerScrubbingSvcInterval = conf.getObject( - DatanodeConfiguration.class).getRecoveringContainerScrubInterval(); + Duration recoveringContainerScrubbingSvcInterval = + dnConf.getRecoveringContainerScrubInterval(); long recoveringContainerScrubbingServiceTimeout = config .getTimeDuration(OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_TIMEOUT, --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
