This is an automated email from the ASF dual-hosted git repository. sammichen pushed a commit to branch HDDS-3630 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 4f58545294e3de92929d2fe814bcb98b98929733 Merge: 24ca66b067 f94b7eb12d Author: Sammi Chen <[email protected]> AuthorDate: Thu Jun 2 11:21:52 2022 +0800 Merge remote-tracking branch 'origin/HDDS-3630' .../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 + .../java/org/apache/hadoop/ozone/OzoneConsts.java | 6 +- .../upgrade/AbstractLayoutVersionManager.java | 12 +- .../common/src/main/resources/ozone-default.xml | 14 + .../apache/hadoop/ozone/HddsDatanodeService.java | 6 +- .../container/common/interfaces/DBHandle.java | 48 ++ .../common/statemachine/DatanodeConfiguration.java | 66 ++ .../common/statemachine/DatanodeStateMachine.java | 11 +- .../commandhandler/DeleteBlocksCommandHandler.java | 90 ++- .../states/endpoint/VersionEndpointTask.java | 68 +- .../container/common/utils/ContainerCache.java | 2 +- .../container/common/utils/DatanodeStoreCache.java | 88 +++ .../container/common/utils/HddsVolumeUtil.java | 230 ++----- .../hadoop/ozone/container/common/utils/RawDB.java | 42 ++ .../container/common/utils/ReferenceCountedDB.java | 25 +- .../container/common/utils/StorageVolumeUtil.java | 202 ++++++ .../ozone/container/common/volume/DbVolume.java | 153 +++++ ...dataVolumeFactory.java => DbVolumeFactory.java} | 35 +- .../ozone/container/common/volume/HddsVolume.java | 407 +++++------ .../container/common/volume/HddsVolumeFactory.java | 37 +- .../container/common/volume/MetadataVolume.java | 5 + .../common/volume/MetadataVolumeFactory.java | 2 +- .../container/common/volume/MutableVolumeSet.java | 11 + .../container/common/volume/StorageVolume.java | 278 +++++++- .../common/volume/StorageVolumeFactory.java | 41 +- .../container/keyvalue/KeyValueContainer.java | 121 ++-- .../container/keyvalue/KeyValueContainerCheck.java | 24 +- .../container/keyvalue/KeyValueContainerData.java | 109 ++- .../KeyValueContainerMetadataInspector.java | 62 +- .../ozone/container/keyvalue/KeyValueHandler.java | 11 +- .../container/keyvalue/TarContainerPacker.java | 15 +- .../container/keyvalue/helpers/BlockUtils.java | 179 ++++- .../helpers/KeyValueContainerLocationUtil.java | 12 +- .../keyvalue/helpers/KeyValueContainerUtil.java | 194 +++--- .../container/keyvalue/impl/BlockManagerImpl.java | 57 +- .../background/BlockDeletingService.java | 95 ++- .../metadata/AbstractDatanodeDBDefinition.java | 11 +- .../container/metadata/AbstractDatanodeStore.java | 20 +- .../metadata/DatanodeSchemaOneDBDefinition.java | 6 +- .../metadata/DatanodeSchemaThreeDBDefinition.java | 158 +++++ .../metadata/DatanodeSchemaTwoDBDefinition.java | 6 +- .../ozone/container/metadata/DatanodeStore.java | 9 +- .../metadata/DatanodeStoreSchemaOneImpl.java | 7 +- .../metadata/DatanodeStoreSchemaThreeImpl.java | 130 ++++ .../metadata/DatanodeStoreSchemaTwoImpl.java | 13 +- .../ozone/container/metadata/DatanodeTable.java | 34 +- .../container/metadata/DeleteTransactionStore.java | 29 + .../metadata/SchemaOneDeletedBlocksTable.java | 8 +- .../ozone/container/ozoneimpl/OzoneContainer.java | 33 +- .../upgrade/DatanodeSchemaV3FinalizeAction.java | 82 +++ .../ScmHAFinalizeUpgradeActionDatanode.java | 2 +- .../upgrade/VersionedDatanodeFeatures.java | 28 +- .../ozone/container/common/ContainerTestUtils.java | 32 + .../container/common/TestBlockDeletingService.java | 167 ++--- .../ozone/container/common/TestContainerCache.java | 2 +- .../container/common/TestDatanodeStoreCache.java | 78 +++ .../common/TestKeyValueContainerData.java | 20 +- .../TestSchemaOneBackwardsCompatibility.java | 150 ++-- .../TestSchemaTwoBackwardsCompatibility.java | 367 ++++++++++ .../common/helpers/TestDatanodeVersionFile.java | 18 +- .../common/impl/TestContainerPersistence.java | 30 +- .../statemachine/TestDatanodeConfiguration.java | 11 + .../container/common/utils/TestHddsVolumeUtil.java | 238 +++++++ .../common/utils/TestStorageVolumeUtil.java | 99 +++ .../container/common/volume/TestDbVolume.java | 172 +++++ .../container/common/volume/TestHddsVolume.java | 164 ++++- .../container/common/volume/TestStorageVolume.java | 83 +++ .../common/volume/TestVolumeSetDiskChecks.java | 30 + .../keyvalue/ContainerTestVersionInfo.java | 79 +++ .../keyvalue/TestKeyValueBlockIterator.java | 82 ++- .../container/keyvalue/TestKeyValueContainer.java | 47 +- .../keyvalue/TestKeyValueContainerCheck.java | 19 +- .../TestKeyValueContainerIntegrityChecks.java | 44 +- .../TestKeyValueContainerMetadataInspector.java | 20 +- .../container/keyvalue/TestTarContainerPacker.java | 82 ++- .../keyvalue/impl/TestBlockManagerImpl.java | 23 +- .../container/ozoneimpl/TestContainerReader.java | 85 ++- .../container/ozoneimpl/TestOzoneContainer.java | 99 +-- .../upgrade/TestDatanodeUpgradeToSchemaV3.java | 759 +++++++++++++++++++++ .../upgrade/TestDatanodeUpgradeToScmHA.java | 5 + .../docs/content/design/dn-merge-rocksdb.md | 29 + .../docs/content/feature/dn-merge-rocksdb.md | 70 ++ .../apache/hadoop/hdds/utils/HddsServerUtil.java | 7 + .../hadoop/hdds/utils/MetadataKeyFilters.java | 1 + .../hdds/utils/db/DBColumnFamilyDefinition.java | 13 + .../hadoop/hdds/utils/db/DBStoreBuilder.java | 2 +- .../hadoop/hdds/utils/db/DumpFileLoader.java | 38 ++ .../hadoop/hdds/utils/db/DumpFileWriter.java | 46 ++ .../hdds/utils/db/FixedLengthStringCodec.java | 50 ++ .../hdds/utils/db/FixedLengthStringUtils.java | 57 ++ .../hadoop/hdds/utils/db/RDBSstFileLoader.java | 64 ++ .../hadoop/hdds/utils/db/RDBSstFileWriter.java | 99 +++ .../hadoop/hdds/utils/db/RDBStoreIterator.java | 54 +- .../org/apache/hadoop/hdds/utils/db/RDBTable.java | 82 ++- .../apache/hadoop/hdds/utils/db/RocksDatabase.java | 16 +- .../org/apache/hadoop/hdds/utils/db/Table.java | 43 +- .../apache/hadoop/hdds/utils/db/TypedTable.java | 44 +- .../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 79 +++ .../hdds/utils/db/TestFixedLengthStringUtils.java | 45 ++ .../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 54 ++ .../hadoop/hdds/utils/db/TestRDBTableStore.java | 226 ++++++ .../src/main/proto/DatanodeClientProtocol.proto | 2 + .../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 + .../hadoop/hdds/scm/server/SCMCertStore.java | 6 +- .../main/resources/webapps/scm/scm-overview.html | 2 +- .../dist/src/main/compose/compatibility/test.sh | 1 + .../non-rolling-upgrade/1.2.1-1.3.0/callback.sh | 2 +- .../smoketest/compatibility/dn-one-rocksdb.robot | 29 + .../ozone/TestStorageContainerManagerHelper.java | 80 +-- .../client/rpc/TestFailureHandlingByClient.java | 14 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 21 +- .../client/rpc/TestValidateBCSIDOnRestart.java | 18 +- .../commandhandler/TestBlockDeletion.java | 25 +- .../TestCloseContainerByPipeline.java | 6 +- .../hadoop/ozone/debug/DBDefinitionFactory.java | 14 +- .../org/apache/hadoop/ozone/debug/DBScanner.java | 3 +- .../apache/hadoop/ozone/debug/PrefixParser.java | 2 +- .../ozone/debug/container/ContainerCommands.java | 3 +- .../containergenerator/GeneratorDatanode.java | 8 +- .../ozone/debug/TestDBDefinitionFactory.java | 13 +- 121 files changed, 6330 insertions(+), 1387 deletions(-) diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java index e817202846,9fc07840da..bcdedf5654 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java @@@ -18,12 -18,11 +18,12 @@@ package org.apache.hadoop.ozone.container.common.volume; - import static org.apache.hadoop.ozone.container.common.HDDSVolumeLayoutVersion.getLatestVersion; - import java.io.File; import java.io.IOException; - import java.util.Properties; + import java.util.List; +import java.util.UUID; + import java.util.concurrent.ThreadLocalRandom; + import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.hdds.annotation.InterfaceAudience; @@@ -65,20 -62,18 +69,21 @@@ public class HddsVolume extends Storage public static final String HDDS_VOLUME_DIR = "hdds"; - private VolumeState state; private final VolumeIOStats volumeIOStats; + private final VolumeInfoMetrics volumeInfoMetrics; - // VERSION file properties - private String storageID; // id of the file system - private String clusterID; // id of the cluster - private String datanodeUuid; // id of the DataNode - private long cTime; // creation time of the file system state - private int layoutVersion; // layout version of the storage data private final AtomicLong committedBytes; // till Open containers become full + // Mentions the type of volume + private final VolumeType type = VolumeType.DATA_VOLUME; + // The dedicated DbVolume that the db instance of this HddsVolume resides. + // This is optional, if null then the db instance resides on this HddsVolume. + private DbVolume dbVolume; + // The subdirectory with storageID as its name, used to build the + // container db path. This is initialized only once together with dbVolume, + // and stored as a member to prevent spawning lots of File objects. + private File dbParentDir; + private AtomicBoolean dbLoaded = new AtomicBoolean(false); /** * Builder for HddsVolume. @@@ -115,12 -98,7 +108,10 @@@ super(b); if (!b.getFailedVolume()) { - this.state = VolumeState.NOT_INITIALIZED; - this.clusterID = b.clusterID; - this.datanodeUuid = b.datanodeUuid; ++ this.setState(VolumeState.NOT_INITIALIZED); this.volumeIOStats = new VolumeIOStats(b.getVolumeRootStr()); + this.volumeInfoMetrics = + new VolumeInfoMetrics(b.getVolumeRootStr(), this); this.committedBytes = new AtomicLong(0); LOG.info("Creating HddsVolume: {} of storage type : {} capacity : {}", @@@ -130,10 -108,7 +121,9 @@@ } else { // Builder is called with failedVolume set, so create a failed volume // HddsVolume Object. ++ this.setState(VolumeState.FAILED); volumeIOStats = null; + volumeInfoMetrics = null; - storageID = UUID.randomUUID().toString(); - state = VolumeState.FAILED; committedBytes = null; } @@@ -288,61 -130,17 +145,28 @@@ return super.getStorageDir(); } - @Override - public String getStorageID() { - return storageID; - } - - public String getClusterID() { - return clusterID; - } - - public String getDatanodeUuid() { - return datanodeUuid; - } - - public long getCTime() { - return cTime; - } - - public int getLayoutVersion() { - return layoutVersion; - } - + public VolumeType getType() { + return type; + } + - public VolumeState getStorageState() { - return state; - } - - public void setState(VolumeState state) { - this.state = state; - } - - public boolean isFailed() { - return (state == VolumeState.FAILED); - } - public VolumeIOStats getVolumeIOStats() { return volumeIOStats; } + public VolumeInfoMetrics getVolumeInfoStats() { + return volumeInfoMetrics; + } + @Override public void failVolume() { - setState(VolumeState.FAILED); super.failVolume(); if (volumeIOStats != null) { volumeIOStats.unregister(); } + if (volumeInfoMetrics != null) { + volumeInfoMetrics.unregister(); + } + closeDbStore(); } @Override @@@ -352,30 -149,7 +175,10 @@@ if (volumeIOStats != null) { volumeIOStats.unregister(); } + if (volumeInfoMetrics != null) { + volumeInfoMetrics.unregister(); + } - } - - /** - * VolumeState represents the different states a HddsVolume can be in. - * NORMAL => Volume can be used for storage - * FAILED => Volume has failed due and can no longer be used for - * storing containers. - * NON_EXISTENT => Volume Root dir does not exist - * INCONSISTENT => Volume Root dir is not empty but VERSION file is - * missing or Volume Root dir is not a directory - * NOT_FORMATTED => Volume Root exists but not formatted(no VERSION file) - * NOT_INITIALIZED => VERSION file exists but has not been verified for - * correctness. - */ - public enum VolumeState { - NORMAL, - FAILED, - NON_EXISTENT, - INCONSISTENT, - NOT_FORMATTED, - NOT_INITIALIZED + closeDbStore(); } /** --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
