This is an automated email from the ASF dual-hosted git repository. erose pushed a commit to branch HDDS-10239-container-reconciliation in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 98369a8343f12479af99db4c83696945d40c1e3d Merge: ab35173e24 51ba4c88ab Author: Ethan Rose <[email protected]> AuthorDate: Wed Aug 7 16:22:02 2024 -0400 Merge branch 'master' into HDDS-10239-container-reconciliation * master: (181 commits) HDDS-11289. Bump docker-maven-plugin to 0.45.0 (#7024) HDDS-11287. Code cleanup in XceiverClientSpi (#7043) HDDS-11283. Refactor KeyValueStreamDataChannel to avoid spurious IDE build issues (#7040) HDDS-11257. Ozone write does not work when http proxy is set for the JVM. (#7036) HDDS-11249. Bump ozone-runner to 20240729-jdk17-1 (#7003) HDDS-10517. Recon - Add a UI component for showing DN decommissioning detailed status and info (#6724) HDDS-11270. [hsync] Add DN layout version (HBASE_SUPPORT/version 8) upgrade test. (#7021) HDDS-11272. Statistics some node status information (#7025) HDDS-11278. Move code out of Hadoop util package (#7028) HDDS-11274. (addendum) Replace Hadoop annotations/configs with Ozone-specific ones HDDS-11274. Replace Hadoop annotations/configs with Ozone-specific ones (#7026) HDDS-11280. Add Synchronize in AbstractCommitWatcher.addAckDataLength (#7032) HDDS-11235. Spare InfoBucket RPC call in FileSystem#mkdir() call. (#6990) HDDS-11273. Bump commons-compress to 1.26.2 (#7023) HDDS-11225. Increase ipc.server.read.threadpool.size (#7007) HDDS-11224. Increase hdds.datanode.handler.count (#7011) HDDS-11259. [hsync] DataNode should verify HBASE_SUPPORT layout version for every PutBlock. (#7012) HDDS-11214. Added config to set rocksDB's max log file size and num of log files (#7014) HDDS-11226. Make ExponentialBackoffPolicy maxRetries configurable (#6985) HDDS-11260. [hsync] Add Ozone Manager protocol version (#7015) ... Conflicts: hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java .../hadoop/hdds/scm/ContainerClientMetrics.java | 42 + .../apache/hadoop/hdds/scm/OzoneClientConfig.java | 33 + .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 12 +- .../hadoop/hdds/scm/XceiverClientManager.java | 2 +- .../hadoop/hdds/scm/XceiverClientMetrics.java | 2 +- .../apache/hadoop/hdds/scm/XceiverClientRatis.java | 4 +- .../hdds/scm/storage/AbstractCommitWatcher.java | 13 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 90 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 651 +++++++++++--- .../apache/hadoop/hdds/scm/storage/BufferPool.java | 186 ++-- .../hadoop/hdds/scm/storage/CommitWatcher.java | 44 +- .../hdds/scm/storage/ECBlockOutputStream.java | 17 +- .../hdds/scm/storage/RatisBlockOutputStream.java | 23 +- .../client/io/BlockInputStreamFactoryImpl.java | 2 +- .../hdds/scm/storage/DummyBlockInputStream.java | 10 +- .../storage/DummyBlockInputStreamWithRetry.java | 7 +- .../hdds/scm/storage/TestBlockInputStream.java | 21 +- .../hadoop/hdds/scm/storage/TestBufferPool.java | 120 ++- .../org/apache/hadoop/hdds/DatanodeVersion.java | 2 + .../org/apache/hadoop/hdds/HddsConfigKeys.java | 2 +- .../java/org/apache/hadoop/hdds/HddsUtils.java | 7 + .../hadoop/hdds/protocol/DatanodeDetails.java | 8 +- .../org/apache/hadoop/hdds/ratis/RatisHelper.java | 6 +- .../hadoop/hdds/ratis/conf/RatisClientConfig.java | 15 + .../RequestTypeDependentRetryPolicyCreator.java | 2 + .../apache/hadoop/hdds/scm/XceiverClientSpi.java | 6 +- .../apache/hadoop/hdds/scm/net/NetConstants.java | 2 +- .../org/apache/hadoop/hdds/scm/net/NodeImpl.java | 2 +- .../ContainerCommandResponseBuilders.java | 35 + .../hadoop/hdds/scm/storage/BlockLocationInfo.java | 10 + .../hdds/scm/storage/ContainerProtocolCalls.java | 43 +- .../security/ssl/PemFileBasedKeyStoresFactory.java | 165 ---- .../hdds/security/ssl/ReloadingX509KeyManager.java | 96 ++- .../security/ssl/ReloadingX509TrustManager.java | 82 +- .../security/token/ContainerTokenIdentifier.java | 2 +- .../security/token/OzoneBlockTokenIdentifier.java | 2 +- .../x509/certificate/client/CertificateClient.java | 13 +- .../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 10 +- .../java/org/apache/hadoop/ozone/OzoneConsts.java | 6 + .../apache/hadoop/ozone/OzoneManagerVersion.java | 1 + .../common/ChunkBufferImplWithByteBuffer.java | 4 +- .../common/ChunkBufferImplWithByteBufferList.java | 41 +- .../hadoop/{ => ozone}/util/CacheMetrics.java | 2 +- .../hadoop/{ => ozone}/util/ClosableIterator.java | 2 +- .../apache/hadoop/{ => ozone}/util/MetricUtil.java | 13 +- .../hadoop/{ => ozone}/util/MutableMinMax.java | 6 +- .../{ => ozone}/util/PerformanceMetrics.java | 2 +- .../util/PerformanceMetricsInitializer.java | 2 +- .../hadoop/{ => ozone}/util/ProtobufUtils.java | 2 +- .../{ => ozone}/util/StringWithByteString.java | 2 +- .../apache/hadoop/{ => ozone}/util/UUIDUtil.java | 2 +- .../java/org/apache/hadoop/util/package-info.java | 23 - .../common/src/main/resources/ozone-default.xml | 32 +- .../hdds/ratis/conf/TestRatisClientConfig.java | 5 + .../hadoop/hdds/utils/TestProtobufUtils.java | 7 +- .../TestChunkBufferImplWithByteBufferList.java | 70 ++ .../ozone/container/ContainerTestHelper.java | 45 +- .../apache/hadoop/ozone/HddsDatanodeService.java | 31 +- .../org/apache/hadoop/ozone/audit/DNAction.java | 1 + .../checksum/ContainerChecksumTreeManager.java | 2 +- .../container/common/helpers/ContainerMetrics.java | 11 +- .../container/common/helpers/DatanodeIdYaml.java | 2 +- .../container/common/impl/HddsDispatcher.java | 60 +- .../common/impl/StorageLocationReport.java | 2 +- .../ozone/container/common/interfaces/Handler.java | 4 + .../common/statemachine/DatanodeStateMachine.java | 29 +- .../common/statemachine/StateContext.java | 18 +- .../container/common/states/DatanodeState.java | 6 + .../states/datanode/RunningDatanodeState.java | 90 +- .../common/transport/server/XceiverServerGrpc.java | 2 +- .../server/ratis/ContainerStateMachine.java | 54 +- .../transport/server/ratis/XceiverServerRatis.java | 11 +- .../common/volume/AvailableSpaceFilter.java | 2 +- .../ozone/container/common/volume/VolumeUsage.java | 72 +- .../container/keyvalue/KeyValueContainer.java | 7 + .../container/keyvalue/KeyValueContainerData.java | 34 + .../ozone/container/keyvalue/KeyValueHandler.java | 132 ++- .../keyvalue/helpers/KeyValueContainerUtil.java | 23 + .../container/keyvalue/impl/BlockManagerImpl.java | 89 +- .../ozone/container/keyvalue/impl/Buffers.java | 115 +++ .../keyvalue/impl/ChunkManagerDispatcher.java | 6 + .../keyvalue/impl/FilePerBlockStrategy.java | 22 + .../keyvalue/impl/KeyValueStreamDataChannel.java | 94 +- .../keyvalue/impl/StreamDataChannelBase.java | 4 +- .../keyvalue/interfaces/BlockManager.java | 3 + .../keyvalue/interfaces/ChunkManager.java | 5 + .../metadata/AbstractDatanodeDBDefinition.java | 7 + .../container/metadata/AbstractDatanodeStore.java | 138 ++- .../metadata/DatanodeSchemaOneDBDefinition.java | 6 + .../metadata/DatanodeSchemaThreeDBDefinition.java | 36 +- .../metadata/DatanodeSchemaTwoDBDefinition.java | 33 +- .../ozone/container/metadata/DatanodeStore.java | 49 ++ .../metadata/DatanodeStoreSchemaThreeImpl.java | 9 +- .../metadata/DatanodeStoreSchemaTwoImpl.java | 2 +- .../DatanodeStoreWithIncrementalChunkList.java | 237 ++++++ .../container/ozoneimpl/ContainerController.java | 23 + .../ozone/container/ozoneimpl/OzoneContainer.java | 4 +- .../replication/GrpcReplicationClient.java | 9 +- .../container/replication/ReplicationServer.java | 6 +- .../common/volume/TestReservedVolumeSpace.java | 26 + .../container/keyvalue/TestKeyValueHandler.java | 8 + .../TestKeyValueHandlerWithUnhealthyContainer.java | 14 + .../keyvalue/impl/TestBlockManagerImpl.java | 186 +++- .../impl/TestKeyValueStreamDataChannel.java | 1 - .../upgrade/TestDatanodeUpgradeToHBaseSupport.java | 332 ++++++++ hadoop-hdds/docs/content/tools/Admin.md | 141 ++- .../hdds/protocol/SecretKeyProtocolDatanode.java | 2 +- .../protocolPB/SecretKeyProtocolDatanodePB.java | 2 +- ...inerLocationProtocolClientSideTranslatorPB.java | 2 +- .../hdds/security/symmetric/ManagedSecretKey.java | 2 +- .../client/DefaultCertificateClient.java | 49 +- .../hdds/security/x509/keys/SecurityUtil.java | 35 - .../apache/hadoop/hdds/utils/FaultInjector.java | 19 + .../org/apache/hadoop/hdds/utils/SignalLogger.java | 17 +- .../hadoop/hdds/utils/db/DBStoreBuilder.java | 4 + .../hadoop/hdds/utils/db/RocksDBConfiguration.java | 30 + .../hadoop/hdds/utils/db/cache/FullTableCache.java | 97 ++- .../security/ssl/TestReloadingX509KeyManager.java | 14 +- .../ssl/TestReloadingX509TrustManager.java | 9 +- ...ctory.java => TestSSLConnectionWithReload.java} | 54 +- .../client/CertificateClientTestImpl.java | 61 +- .../hadoop/hdds/utils/db/cache/TestTableCache.java | 34 +- .../src/main/proto/DatanodeClientProtocol.proto | 31 +- .../interface-client/src/main/proto/hdds.proto | 1 + .../db/managed/ManagedRawSSTFileIterator.java | 2 +- .../ozone/compaction/log/CompactionFileInfo.java | 2 +- .../ozone/compaction/log/CompactionLogEntry.java | 2 +- .../ozone/rocksdb/util/SstFileSetReader.java | 2 +- .../container/AbstractContainerReportHandler.java | 12 +- .../hdds/scm/container/ContainerManager.java | 10 + .../hdds/scm/container/ContainerManagerImpl.java | 15 + .../hdds/scm/container/ContainerStateManager.java | 12 + .../scm/container/ContainerStateManagerImpl.java | 23 + .../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 8 +- .../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 11 +- .../hdds/scm/ha/InterSCMGrpcProtocolService.java | 6 +- .../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 13 + .../hdds/scm/node/NodeDecommissionManager.java | 6 +- .../hadoop/hdds/scm/node/NodeManagerMXBean.java | 5 + .../hadoop/hdds/scm/node/NodeStateManager.java | 52 ++ .../hadoop/hdds/scm/node/SCMNodeManager.java | 92 ++ .../SecretKeyProtocolServerSideTranslatorPB.java | 2 +- ...inerLocationProtocolServerSideTranslatorPB.java | 2 +- .../main/resources/webapps/scm/scm-overview.html | 56 ++ .../src/main/resources/webapps/scm/scm.js | 39 + .../scm/container/TestContainerManagerImpl.java | 58 ++ .../scm/container/TestContainerReportHandler.java | 106 +++ .../scm/container/TestContainerStateManager.java | 44 + .../TestContainerBalancerDatanodeNodeLimit.java | 275 +++++- .../balancer/TestContainerBalancerTask.java | 259 ------ .../scm/ha/TestInterSCMGrpcProtocolService.java | 86 +- .../java/org/apache/hadoop/test/package-info.java | 22 - .../{hadoop => ozone}/test/JacocoServer.java | 2 +- .../java/org/apache/ozone/test/MetricsAsserts.java | 2 +- .../ozone/client/io/BlockOutputStreamEntry.java | 3 +- .../client/io/BlockOutputStreamEntryPool.java | 35 +- .../hadoop/ozone/client/io/KeyInputStream.java | 10 +- .../hadoop/ozone/client/io/KeyOutputStream.java | 52 +- .../ozone/client/protocol/ClientProtocol.java | 24 + .../apache/hadoop/ozone/client/rpc/RpcClient.java | 32 +- .../hadoop/ozone/client/MockDatanodeStorage.java | 131 ++- .../hadoop/ozone/client/MockOmTransport.java | 50 +- .../hadoop/ozone/client/MockXceiverClientSpi.java | 37 +- .../TestBlockOutputStreamIncrementalPutBlock.java | 160 ++++ .../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 + .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 8 + .../hadoop/ozone/om/exceptions/OMException.java | 6 +- .../{OpenKeySession.java => LeaseKeyInfo.java} | 27 +- .../ozone/om/helpers/ListOpenFilesResult.java | 115 +++ .../hadoop/ozone/om/helpers/OpenKeySession.java | 3 + .../ozone/om/protocol/OzoneManagerProtocol.java | 32 +- .../ozone/om/protocolPB/GrpcOmTransport.java | 1 + ...OzoneManagerProtocolClientSideTranslatorPB.java | 71 +- hadoop-ozone/dist/pom.xml | 2 +- .../dist/src/main/compose/ozone/docker-config | 2 + hadoop-ozone/dist/src/main/compose/test-all.sh | 2 +- hadoop-ozone/dist/src/shell/ozone/ozone | 18 +- .../dist/src/shell/ozone/ozone-functions.sh | 8 + hadoop-ozone/fault-injection-test/pom.xml | 8 + .../fs/ozone/AbstractOzoneFileSystemTest.java | 1 + .../ozone/AbstractOzoneFileSystemTestWithFSO.java | 3 +- .../ozone/AbstractRootedOzoneFileSystemTest.java | 1 + .../AbstractRootedOzoneFileSystemTestWithFSO.java | 4 +- .../ozone/TestDirectoryDeletingServiceWithFSO.java | 80 ++ .../java/org/apache/hadoop/fs/ozone/TestHSync.java | 941 +++++++++++++++++++-- .../apache/hadoop/fs/ozone/TestLeaseRecovery.java | 487 ++++++++++- .../hadoop/fs/ozone/TestOzoneFSInputStream.java | 123 +++ .../ozone/contract/AbstractOzoneContractTest.java | 44 + .../hadoop/hdds/scm/storage/TestCommitWatcher.java | 13 - .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 30 +- .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 18 + .../org/apache/hadoop/ozone/OzoneTestUtils.java | 2 +- .../java/org/apache/hadoop/ozone/TestDataUtil.java | 42 + .../client/rpc/TestBlockDataStreamOutput.java | 27 + .../ozone/client/rpc/TestBlockOutputStream.java | 94 +- .../rpc/TestBlockOutputStreamWithFailures.java | 44 +- .../ozone/client/rpc/TestDatanodeVersion.java | 143 ++++ .../rpc/TestOzoneClientMultipartUploadWithFSO.java | 5 +- .../ozone/client/rpc/TestSecureOzoneRpcClient.java | 150 +++- .../container/TestContainerReportHandling.java | 156 ++++ .../TestContainerReportHandlingWithHA.java | 168 ++++ .../apache/hadoop/ozone/container/TestHelper.java | 21 +- .../commandhandler/TestFinalizeBlock.java | 307 +++++++ .../hadoop/ozone/debug/TestLeaseRecoverer.java | 25 +- .../hadoop/ozone/freon/TestDNRPCLoadGenerator.java | 2 +- ...CLoadGenerator.java => TestHsyncGenerator.java} | 94 +- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 48 +- .../hadoop/ozone/om/TestObjectStoreWithFSO.java | 5 +- .../ozone/om/TestOmContainerLocationCache.java | 36 +- .../org/apache/hadoop/ozone/om/TestOmMetrics.java | 9 +- .../snapshot/TestSnapshotBackgroundServices.java | 3 +- .../TestReconInsightsForDeletedDirectories.java | 2 +- .../hadoop/ozone/shell/TestOzoneShellHA.java | 387 ++++++++- .../ozone/shell/TestOzoneShellHAWithFSO.java | 2 + .../org/apache/hadoop/utils/FaultInjectorImpl.java | 95 +++ .../src/main/proto/OmClientProtocol.proto | 35 +- .../apache/hadoop/ozone/om/OMMetadataManager.java | 52 +- .../org/apache/hadoop/ozone/audit/OMAction.java | 5 +- .../hadoop/ozone/om/GrpcOzoneManagerServer.java | 4 +- .../org/apache/hadoop/ozone/om/KeyManager.java | 3 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 12 +- .../java/org/apache/hadoop/ozone/om/OMMetrics.java | 17 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 120 ++- .../apache/hadoop/ozone/om/OmMetadataReader.java | 2 +- .../apache/hadoop/ozone/om/OmSnapshotManager.java | 7 + .../org/apache/hadoop/ozone/om/OzoneManager.java | 140 ++- .../hadoop/ozone/om/OzoneManagerServiceGrpc.java | 2 +- .../java/org/apache/hadoop/ozone/om/ScmClient.java | 2 +- .../hadoop/ozone/om/SstFilteringService.java | 78 +- .../hadoop/ozone/om/helpers/OMAuditLogger.java | 203 +++++ .../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 4 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 2 +- .../om/ratis/utils/OzoneManagerRatisUtils.java | 5 +- .../hadoop/ozone/om/request/OMClientRequest.java | 25 +- .../hadoop/ozone/om/request/RequestAuditor.java | 6 +- .../om/request/bucket/OMBucketCreateRequest.java | 2 +- .../om/request/bucket/OMBucketDeleteRequest.java | 2 +- .../om/request/bucket/OMBucketSetOwnerRequest.java | 2 +- .../request/bucket/OMBucketSetPropertyRequest.java | 2 +- .../request/bucket/acl/OMBucketAddAclRequest.java | 2 +- .../bucket/acl/OMBucketRemoveAclRequest.java | 2 +- .../request/bucket/acl/OMBucketSetAclRequest.java | 2 +- .../om/request/file/OMDirectoryCreateRequest.java | 2 +- .../file/OMDirectoryCreateRequestWithFSO.java | 2 +- .../ozone/om/request/file/OMFileCreateRequest.java | 2 +- .../request/file/OMFileCreateRequestWithFSO.java | 4 +- .../ozone/om/request/file/OMFileRequest.java | 10 +- .../om/request/file/OMRecoverLeaseRequest.java | 167 ++-- .../om/request/key/OMAllocateBlockRequest.java | 14 +- .../request/key/OMAllocateBlockRequestWithFSO.java | 19 +- .../key/OMDirectoriesPurgeRequestWithFSO.java | 33 +- .../ozone/om/request/key/OMKeyCommitRequest.java | 166 +++- .../om/request/key/OMKeyCommitRequestWithFSO.java | 139 ++- .../ozone/om/request/key/OMKeyCreateRequest.java | 2 +- .../om/request/key/OMKeyCreateRequestWithFSO.java | 4 +- .../ozone/om/request/key/OMKeyDeleteRequest.java | 13 +- .../om/request/key/OMKeyDeleteRequestWithFSO.java | 11 +- .../ozone/om/request/key/OMKeyRenameRequest.java | 2 +- .../om/request/key/OMKeyRenameRequestWithFSO.java | 2 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 2 +- .../ozone/om/request/key/OMKeySetTimesRequest.java | 2 +- .../ozone/om/request/key/OMKeysDeleteRequest.java | 23 +- .../ozone/om/request/key/OMKeysRenameRequest.java | 2 +- .../om/request/key/OmKeysDeleteRequestWithFSO.java | 17 +- .../om/request/key/acl/OMKeyAddAclRequest.java | 2 +- .../request/key/acl/OMKeyAddAclRequestWithFSO.java | 2 +- .../om/request/key/acl/OMKeyRemoveAclRequest.java | 2 +- .../key/acl/OMKeyRemoveAclRequestWithFSO.java | 2 +- .../om/request/key/acl/OMKeySetAclRequest.java | 2 +- .../request/key/acl/OMKeySetAclRequestWithFSO.java | 2 +- .../key/acl/prefix/OMPrefixAddAclRequest.java | 2 +- .../key/acl/prefix/OMPrefixRemoveAclRequest.java | 2 +- .../key/acl/prefix/OMPrefixSetAclRequest.java | 2 +- .../S3ExpiredMultipartUploadsAbortRequest.java | 2 +- .../S3InitiateMultipartUploadRequest.java | 2 +- .../S3InitiateMultipartUploadRequestWithFSO.java | 2 +- .../multipart/S3MultipartUploadAbortRequest.java | 2 +- .../S3MultipartUploadCommitPartRequest.java | 2 +- .../S3MultipartUploadCompleteRequest.java | 6 +- .../S3MultipartUploadCompleteRequestWithFSO.java | 4 +- .../om/request/s3/security/OMSetSecretRequest.java | 2 +- .../om/request/s3/security/S3GetSecretRequest.java | 2 +- .../request/s3/security/S3RevokeSecretRequest.java | 2 +- .../s3/tenant/OMTenantAssignAdminRequest.java | 2 +- .../tenant/OMTenantAssignUserAccessIdRequest.java | 2 +- .../request/s3/tenant/OMTenantCreateRequest.java | 6 +- .../request/s3/tenant/OMTenantDeleteRequest.java | 9 +- .../s3/tenant/OMTenantRevokeAdminRequest.java | 2 +- .../tenant/OMTenantRevokeUserAccessIdRequest.java | 2 +- .../security/OMCancelDelegationTokenRequest.java | 4 +- .../security/OMGetDelegationTokenRequest.java | 4 +- .../security/OMRenewDelegationTokenRequest.java | 4 +- .../request/snapshot/OMSnapshotCreateRequest.java | 2 +- .../request/snapshot/OMSnapshotDeleteRequest.java | 2 +- .../request/snapshot/OMSnapshotRenameRequest.java | 2 +- .../om/request/upgrade/OMCancelPrepareRequest.java | 2 +- .../request/upgrade/OMFinalizeUpgradeRequest.java | 2 +- .../ozone/om/request/upgrade/OMPrepareRequest.java | 2 +- .../ozone/om/request/util/OmKeyHSyncUtil.java | 56 ++ .../om/request/volume/OMVolumeCreateRequest.java | 2 +- .../om/request/volume/OMVolumeDeleteRequest.java | 2 +- .../om/request/volume/OMVolumeSetOwnerRequest.java | 2 +- .../om/request/volume/OMVolumeSetQuotaRequest.java | 2 +- .../request/volume/acl/OMVolumeAddAclRequest.java | 2 +- .../volume/acl/OMVolumeRemoveAclRequest.java | 2 +- .../request/volume/acl/OMVolumeSetAclRequest.java | 2 +- .../om/response/file/OMRecoverLeaseResponse.java | 21 +- .../key/OMDirectoriesPurgeResponseWithFSO.java | 11 +- .../ozone/om/response/key/OMKeyCommitResponse.java | 29 +- .../response/key/OMKeyCommitResponseWithFSO.java | 19 +- .../ozone/om/response/key/OMKeyDeleteResponse.java | 28 +- .../response/key/OMKeyDeleteResponseWithFSO.java | 18 +- .../om/response/key/OMKeysDeleteResponse.java | 20 +- .../response/key/OMKeysDeleteResponseWithFSO.java | 13 +- .../response/snapshot/OMSnapshotPurgeResponse.java | 32 +- .../ozone/om/service/OpenKeyCleanupService.java | 17 +- .../ozone/om/service/SnapshotDeletingService.java | 7 +- .../hadoop/ozone/om/snapshot/PersistentList.java | 2 +- .../hadoop/ozone/om/snapshot/PersistentMap.java | 2 +- .../hadoop/ozone/om/snapshot/PersistentSet.java | 2 +- .../ozone/om/snapshot/RocksDbPersistentList.java | 2 +- .../ozone/om/snapshot/RocksDbPersistentMap.java | 2 +- .../ozone/om/snapshot/RocksDbPersistentSet.java | 2 +- .../ozone/om/snapshot/SnapshotDiffManager.java | 2 +- .../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 3 +- ...OzoneManagerProtocolServerSideTranslatorPB.java | 12 +- .../protocolPB/OzoneManagerRequestHandler.java | 56 +- .../apache/hadoop/ozone/om/TestChunkStreams.java | 2 +- .../hadoop/ozone/om/TestOmMetadataManager.java | 153 +++- .../om/ratis/TestOzoneManagerStateMachine.java | 42 + .../ozone/om/request/OMRequestTestUtils.java | 2 +- .../om/request/file/TestOMRecoverLeaseRequest.java | 391 ++++++++- .../om/request/key/TestOMKeyCommitRequest.java | 12 +- .../ozone/om/request/key/TestOMKeyRequest.java | 11 + ...estS3InitiateMultipartUploadRequestWithFSO.java | 3 +- .../om/response/key/TestOMKeyCommitResponse.java | 8 +- .../key/TestOMKeyCommitResponseWithFSO.java | 4 +- .../om/response/key/TestOMKeysDeleteResponse.java | 2 +- .../key/TestOMKeysDeleteResponseWithFSO.java | 2 +- .../om/service/TestOpenKeyCleanupService.java | 160 +++- .../om/service/TestSnapshotDeletingService.java | 19 +- .../ozone/om/snapshot/SnapshotTestUtils.java | 2 +- .../om/snapshot/TestRocksDbPersistentList.java | 2 +- .../om/snapshot/TestRocksDbPersistentMap.java | 2 +- .../om/snapshot/TestRocksDbPersistentSet.java | 2 +- .../ozone/om/snapshot/TestSnapshotDiffManager.java | 2 +- .../ozone/om/snapshot/TestSstFilteringService.java | 14 +- .../hadoop/fs/ByteBufferPositionedReadable.java | 92 ++ .../fs/ozone/BasicOzoneClientAdapterImpl.java | 104 ++- .../ozone/BasicRootedOzoneClientAdapterImpl.java | 261 ++++-- .../fs/ozone/BasicRootedOzoneFileSystem.java | 44 +- .../hadoop/fs/ozone/CapableOzoneFSInputStream.java | 1 + .../fs/ozone/LeaseRecoveryClientDNHandler.java | 123 +++ .../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 9 +- .../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 49 +- .../org/apache/hadoop/fs/ozone/OzoneFsDelete.java | 4 +- .../hadoop/fs/ozone/OzonePathCapabilities.java | 1 + .../java/org/apache/hadoop/fs/ozone/Statistic.java | 6 +- .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 37 +- .../hadoop/fs/ozone/RootedOzoneFileSystem.java | 37 +- .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 39 +- .../hadoop/fs/ozone/RootedOzoneFileSystem.java | 44 +- .../ozone/recon/api/AccessHeatMapEndpoint.java | 11 + .../ozone/recon/api/types/HealthCheckResponse.java | 71 ++ .../ozone/recon/heatmap/HeatMapServiceImpl.java | 10 + .../ozone/recon/heatmap/IHeatMapProvider.java | 11 + .../spi/impl/OzoneManagerServiceProviderImpl.java | 43 + .../webapps/recon/ozone-recon-web/api/db.json | 210 ++++- .../webapps/recon/ozone-recon-web/api/routes.json | 13 +- .../src/components/overviewCard/overviewCard.tsx | 6 +- .../recon/ozone-recon-web/src/utils/common.tsx | 16 +- .../src/views/datanodes/datanodes.tsx | 81 +- .../src/views/datanodes/decommissionSummary.tsx | 121 +++ .../src/views/diskUsage/diskUsage.tsx | 286 ++++--- .../src/views/overview/overview.tsx | 32 +- .../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 2 +- .../hadoop/ozone/client/ClientProtocolStub.java | 13 + .../ozone/admin/om/ListOpenFilesSubCommand.java | 248 ++++++ .../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 1 + .../hadoop/ozone/freon/DNRPCLoadGenerator.java | 14 +- .../freon/FollowerAppendLogEntryGenerator.java | 3 +- .../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +- .../apache/hadoop/ozone/freon/HsyncGenerator.java | 154 ++++ .../ozone/freon/LeaderAppendLogEntryGenerator.java | 2 +- .../containergenerator/GeneratorDatanode.java | 6 +- .../org/apache/hadoop/test/OzoneTestDriver.java | 61 -- pom.xml | 22 +- 388 files changed, 12437 insertions(+), 2923 deletions(-) diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java index e170bf8591,f7a38e3dec..be594062d0 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java @@@ -40,8 -40,8 +40,9 @@@ public enum DNAction implements AuditAc CLOSE_CONTAINER, GET_COMMITTED_BLOCK_LENGTH, STREAM_INIT, + FINALIZE_BLOCK, - ECHO; + ECHO, + GET_CONTAINER_MERKLE_TREE; @Override public String getAction() { diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerChecksumTreeManager.java index aa2edfdef2,0000000000..7042531f57 mode 100644,000000..100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerChecksumTreeManager.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerChecksumTreeManager.java @@@ -1,220 -1,0 +1,220 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.container.checksum; + +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.ozone.container.common.impl.ContainerData; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Collection; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.Lock; + +import com.google.common.util.concurrent.Striped; +import org.apache.hadoop.hdds.utils.SimpleStriped; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + - import static org.apache.hadoop.util.MetricUtil.captureLatencyNs; ++import static org.apache.hadoop.ozone.util.MetricUtil.captureLatencyNs; + +/** + * This class coordinates reading and writing Container checksum information for all containers. + */ +public class ContainerChecksumTreeManager { + + private static final Logger LOG = LoggerFactory.getLogger(ContainerChecksumTreeManager.class); + + // Used to coordinate reads and writes to each container's checksum file. + // Each container ID is mapped to a stripe. + private final Striped<ReadWriteLock> fileLock; + private final ContainerMerkleTreeMetrics metrics; + + /** + * Creates one instance that should be used to coordinate all container checksum info within a datanode. + */ + public ContainerChecksumTreeManager(ConfigurationSource conf) { + fileLock = SimpleStriped.readWriteLock( + conf.getObject(DatanodeConfiguration.class).getContainerChecksumLockStripes(), true); + // TODO: TO unregister metrics on stop. + metrics = ContainerMerkleTreeMetrics.create(); + } + + /** + * Writes the specified container merkle tree to the specified container's checksum file. + * The data merkle tree within the file is replaced with the {@code tree} parameter, but all other content of the + * file remains unchanged. + * Concurrent writes to the same file are coordinated internally. + */ + public void writeContainerDataTree(ContainerData data, ContainerMerkleTree tree) throws IOException { + Lock writeLock = getWriteLock(data.getContainerID()); + writeLock.lock(); + try { + ContainerProtos.ContainerChecksumInfo newChecksumInfo = read(data).toBuilder() + .setContainerMerkleTree(captureLatencyNs(metrics.getCreateMerkleTreeLatencyNS(), tree::toProto)) + .build(); + write(data, newChecksumInfo); + LOG.debug("Data merkle tree for container {} updated", data.getContainerID()); + } finally { + writeLock.unlock(); + } + } + + /** + * Adds the specified blocks to the list of deleted blocks specified in the container's checksum file. + * All other content of the file remains unchanged. + * Concurrent writes to the same file are coordinated internally. + */ + public void markBlocksAsDeleted(KeyValueContainerData data, Collection<Long> deletedBlockIDs) throws IOException { + Lock writeLock = getWriteLock(data.getContainerID()); + writeLock.lock(); + try { + ContainerProtos.ContainerChecksumInfo.Builder checksumInfoBuilder = read(data).toBuilder(); + // Although the persisted block list should already be sorted, we will sort it here to make sure. + // This will automatically fix any bugs in the persisted order that may show up. + SortedSet<Long> sortedDeletedBlockIDs = new TreeSet<>(checksumInfoBuilder.getDeletedBlocksList()); + sortedDeletedBlockIDs.addAll(deletedBlockIDs); + + checksumInfoBuilder + .clearDeletedBlocks() + .addAllDeletedBlocks(sortedDeletedBlockIDs) + .build(); + write(data, checksumInfoBuilder.build()); + LOG.debug("Deleted block list for container {} updated", data.getContainerID()); + } finally { + writeLock.unlock(); + } + } + + public ContainerDiff diff(KeyValueContainerData thisContainer, ContainerProtos.ContainerChecksumInfo otherInfo) + throws IOException { + // TODO HDDS-10928 compare the checksum info of the two containers and return a summary. + // Callers can act on this summary to repair their container replica using the peer's replica. + // This method will use the read lock, which is unused in the current implementation. + return new ContainerDiff(); + } + + /** + * Returns the container checksum tree file for the specified container without deserializing it. + */ + public static File getContainerChecksumFile(ContainerData data) { + return new File(data.getMetadataPath(), data.getContainerID() + ".tree"); + } + + private Lock getReadLock(long containerID) { + return fileLock.get(containerID).readLock(); + } + + private Lock getWriteLock(long containerID) { + return fileLock.get(containerID).writeLock(); + } + + private ContainerProtos.ContainerChecksumInfo read(ContainerData data) throws IOException { + long containerID = data.getContainerID(); + Lock readLock = getReadLock(containerID); + readLock.lock(); + try { + File checksumFile = getContainerChecksumFile(data); + // If the checksum file has not been created yet, return an empty instance. + // Since all writes happen as part of an atomic read-modify-write cycle that requires a write lock, two empty + // instances for the same container obtained only under the read lock will not conflict. + if (!checksumFile.exists()) { + LOG.debug("No checksum file currently exists for container {} at the path {}. Returning an empty instance.", + containerID, checksumFile); + return ContainerProtos.ContainerChecksumInfo.newBuilder() + .setContainerID(containerID) + .build(); + } + try (FileInputStream inStream = new FileInputStream(checksumFile)) { + return captureLatencyNs(metrics.getReadContainerMerkleTreeLatencyNS(), + () -> ContainerProtos.ContainerChecksumInfo.parseFrom(inStream)); + } + } catch (IOException ex) { + metrics.incrementMerkleTreeReadFailures(); + throw new IOException("Error occurred when reading container merkle tree for containerID " + + data.getContainerID(), ex); + } finally { + readLock.unlock(); + } + } + + private void write(ContainerData data, ContainerProtos.ContainerChecksumInfo checksumInfo) throws IOException { + Lock writeLock = getWriteLock(data.getContainerID()); + writeLock.lock(); + try (FileOutputStream outStream = new FileOutputStream(getContainerChecksumFile(data))) { + captureLatencyNs(metrics.getWriteContainerMerkleTreeLatencyNS(), + () -> checksumInfo.writeTo(outStream)); + } catch (IOException ex) { + metrics.incrementMerkleTreeWriteFailures(); + throw new IOException("Error occurred when writing container merkle tree for containerID " + + data.getContainerID(), ex); + } finally { + writeLock.unlock(); + } + } + + public ByteString getContainerChecksumInfo(KeyValueContainerData data) + throws IOException { + long containerID = data.getContainerID(); + Lock readLock = getReadLock(containerID); + readLock.lock(); + try { + File checksumFile = getContainerChecksumFile(data); + + try (FileInputStream inStream = new FileInputStream(checksumFile)) { + return ByteString.readFrom(inStream); + } catch (FileNotFoundException ex) { + // TODO: Build the container checksum tree when it doesn't exist. + LOG.debug("No checksum file currently exists for container {} at the path {}. Returning an empty instance.", + containerID, checksumFile, ex); + } catch (IOException ex) { + throw new IOException("Error occured when reading checksum file for container " + containerID + + " at the path " + checksumFile, ex); + } + return ByteString.EMPTY; + } finally { + readLock.unlock(); + } + } + + @VisibleForTesting + public ContainerMerkleTreeMetrics getMetrics() { + return this.metrics; + } + + /** + * This class represents the difference between our replica of a container and a peer's replica of a container. + * It summarizes the operations we need to do to reconcile our replica with the peer replica it was compared to. + * + * TODO HDDS-10928 + */ + public static class ContainerDiff { + public ContainerDiff() { + + } + } +} diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java index d7852ac972,cac4df73cc..16f8de4818 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java @@@ -823,8 -828,8 +828,9 @@@ public class HddsDispatcher implements case CloseContainer : return DNAction.CLOSE_CONTAINER; case GetCommittedBlockLength : return DNAction.GET_COMMITTED_BLOCK_LENGTH; case StreamInit : return DNAction.STREAM_INIT; + case FinalizeBlock : return DNAction.FINALIZE_BLOCK; case Echo : return DNAction.ECHO; + case GetContainerMerkleTree : return DNAction.GET_CONTAINER_MERKLE_TREE; default : LOG.debug("Invalid command type - {}", cmdType); return null; diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java index c1a623dbb9,402e1be4cd..9b888a9c1e --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java @@@ -145,7 -146,7 +151,8 @@@ public class KeyValueHandler extends Ha private final boolean validateChunkChecksumData; // A striped lock that is held during container creation. private final Striped<Lock> containerCreationLocks; + private final ContainerChecksumTreeManager checksumManager; + private static FaultInjector injector; public KeyValueHandler(ConfigurationSource config, String datanodeId, @@@ -283,10 -283,10 +290,12 @@@ return handler.handleGetSmallFile(request, kvContainer); case GetCommittedBlockLength: return handler.handleGetCommittedBlockLength(request, kvContainer); + case FinalizeBlock: + return handler.handleFinalizeBlock(request, kvContainer); case Echo: return handler.handleEcho(request, kvContainer); + case GetContainerMerkleTree: + return handler.handleGetContainerMerkleTree(request, kvContainer); default: return null; } diff --cc hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto index 8e58458d52,6cfae24d41..4533a43403 --- a/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto +++ b/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto @@@ -103,8 -105,9 +105,9 @@@ enum Type StreamInit = 19; StreamWrite = 20; - Echo = 21; - GetContainerMerkleTree = 22; - + FinalizeBlock = 21; + Echo = 22; ++ GetContainerMerkleTree = 23; } @@@ -210,8 -214,9 +214,9 @@@ message ContainerCommandRequestProto optional string encodedToken = 23; optional uint32 version = 24; - optional EchoRequestProto echo = 25; - optional GetContainerMerkleTreeRequestProto getContainerMerkleTree = 26; - + optional FinalizeBlockRequestProto finalizeBlock = 25; + optional EchoRequestProto echo = 26; ++ optional GetContainerMerkleTreeRequestProto getContainerMerkleTree = 27; } message ContainerCommandResponseProto { @@@ -240,10 -245,11 +245,10 @@@ optional PutSmallFileResponseProto putSmallFile = 19; optional GetSmallFileResponseProto getSmallFile = 20; -- - optional GetCommittedBlockLengthResponseProto getCommittedBlockLength = 21; - optional EchoResponseProto echo = 22; - optional GetContainerMerkleTreeResponseProto getContainerMerkleTree = 23; + optional GetCommittedBlockLengthResponseProto getCommittedBlockLength = 21; - + optional FinalizeBlockResponseProto finalizeBlock = 22; + optional EchoResponseProto echo = 23; ++ optional GetContainerMerkleTreeResponseProto getContainerMerkleTree = 24; } message ContainerDataProto { diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java index 8b77e30013,7c3666ad61..f7a731fe11 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java @@@ -67,7 -66,7 +67,8 @@@ import java.util.stream.Collectors import java.util.stream.Stream; import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; +import static org.junit.jupiter.api.Assertions.assertTrue; + import static org.apache.hadoop.hdds.scm.HddsTestUtils.getContainerReports; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.mockito.Mockito.doAnswer; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
