This is an automated email from the ASF dual-hosted git repository.
chungen0126 pushed a commit to branch HDDS-11233
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-11233 by this push:
new 93069b8a9ec HDDS-15239. Support persist StorageType on Datanode
Container YAML (#10246)
93069b8a9ec is described below
commit 93069b8a9ecfe6b3e6e28c8e43feb86454dacdd0
Author: XiChen <[email protected]>
AuthorDate: Mon May 18 21:57:58 2026 +0800
HDDS-15239. Support persist StorageType on Datanode Container YAML (#10246)
---
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../container/common/helpers/ContainerUtils.java | 11 +---
.../ozone/container/common/impl/ContainerData.java | 32 +++++++--
.../container/common/impl/ContainerDataYaml.java | 40 +++++++-----
.../common/impl/StorageLocationReport.java | 32 +++++----
.../common/report/NodeReportPublisher.java | 2 +-
.../container/common/report/ReportPublisher.java | 2 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 2 +-
.../common/TestKeyValueContainerData.java | 6 +-
.../TestSchemaOneBackwardsCompatibility.java | 7 +-
.../common/impl/TestContainerDataYaml.java | 75 ++++++++++++++++++++--
11 files changed, 154 insertions(+), 56 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 151022823a1..2656668309a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -243,6 +243,7 @@ public final class OzoneConsts {
public static final String ORIGIN_NODE_ID = "originNodeId";
public static final String SCHEMA_VERSION = "schemaVersion";
public static final String REPLICA_INDEX = "replicaIndex";
+ public static final String CONTAINER_STORAGE_TYPE = "storageType";
// Supported .container datanode schema versions.
// Since containers in older schema versions are currently not reformatted to
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
index 7d16546fb69..dde4588806f 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
@@ -54,14 +54,12 @@
import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
-import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.yaml.snakeyaml.Yaml;
/**
* A mix of helper functions for containers.
@@ -247,11 +245,8 @@ public static void
verifyContainerFileChecksum(ContainerData containerData,
if (enabled) {
String storedChecksum = containerData.getContainerFileChecksum();
- Yaml yaml = ContainerDataYaml.getYamlForContainerType(
- containerData.getContainerType(),
- containerData instanceof KeyValueContainerData &&
- ((KeyValueContainerData)containerData).getReplicaIndex() > 0);
- containerData.computeAndSetContainerFileChecksum(yaml);
+ // Compute checksum (storageTypeis automatically excluded for rollback
compatibility)
+ containerData.computeAndSetContainerFileChecksum();
String computedChecksum = containerData.getContainerFileChecksum();
if (storedChecksum == null || !storedChecksum.equals(computedChecksum)) {
@@ -375,7 +370,7 @@ public static void assertSpaceAvailability(long
containerId, HddsVolume volume,
+ currentUsage + ", minimum free space spared=" + spared,
DISK_OUT_OF_SPACE);
}
}
-
+
public static long getPendingDeletionBytes(ContainerData containerData) {
if (containerData.getContainerType()
.equals(ContainerProtos.ContainerType.KeyValueContainer)) {
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
index 31b890fdbd3..5e6bba1189a 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
@@ -43,12 +43,14 @@
import java.util.Optional;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto;
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.ratis.util.Preconditions;
import org.yaml.snakeyaml.Yaml;
@@ -114,6 +116,7 @@ public abstract class ContainerData {
public static final Charset CHARSET_ENCODING = StandardCharsets.UTF_8;
public static final String ZERO_CHECKSUM = new String(new byte[64],
CHARSET_ENCODING);
+ private StorageType storageType;
// Common Fields need to be stored in .container file.
protected static final List<String> YAML_FIELDS =
@@ -162,6 +165,7 @@ protected ContainerData(ContainerData source) {
source.getLayoutVersion(), source.getMaxSize(),
source.getOriginPipelineId(), source.getOriginNodeId());
replicaIndex = source.getReplicaIndex();
+ storageType = source.getStorageType();
}
/**
@@ -284,6 +288,14 @@ public Map<String, String> getMetadata() {
return Collections.unmodifiableMap(this.metadata);
}
+ public void setStorageType(StorageType type) {
+ storageType = type;
+ }
+
+ public StorageType getStorageType() {
+ return storageType;
+ }
+
/**
* Set metadata.
* We should hold the container lock before updating the metadata as this
@@ -516,22 +528,32 @@ public String getOriginNodeId() {
}
/**
- * Compute the checksum for ContainerData using the specified Yaml (based
- * on ContainerType) and set the checksum.
+ * Compute the checksum for ContainerData and set the checksum.
*
* Checksum of ContainerData is calculated by setting the
* {@link ContainerData#checksum} field to a 64-byte array with all 0's -
* {@link ContainerData#ZERO_CHECKSUM}. After the checksum is calculated,
* the checksum field is updated with this value.
*
- * @param yaml Yaml for ContainerType to get the ContainerData as Yaml String
* @throws IOException
*/
- public void computeAndSetContainerFileChecksum(Yaml yaml) throws IOException
{
+ public void computeAndSetContainerFileChecksum() throws IOException {
// Set checksum to dummy value - 0 byte array, to calculate the checksum
// of rest of the data.
this.checksum = ZERO_CHECKSUM;
+ // Create Yaml for checksum calculation that excludes storageType for
backward compatibility
+ boolean withReplicaIndex = this instanceof KeyValueContainerData &&
+ ((KeyValueContainerData) this).getReplicaIndex() > 0;
+ // IMPORTANT: Pass null for storageType to ensure rollback compatibility.
+ // Ozone calculates checksum by reading the YAML file into a Java object,
then converting
+ // it back to YAML string (not directly using the YAML file content).
+ // If storageType participates in checksum calculation, when rolling back
to older versions that don't
+ // support storageType, they would recalculate checksum without
storageType field
+ // (as it's unknown to them), causing checksum mismatch and validation
failure.
+ Yaml yaml = ContainerDataYaml.getYamlForContainerType(
+ this.containerType, withReplicaIndex, null);
+
// Dump yaml data into a string to compute its checksum
String containerDataYamlStr = yaml.dump(this);
@@ -588,7 +610,7 @@ public boolean needsDataChecksum() {
* - writeCount += 1 (one operation)
* - usedSpace/committedBytes NOT updated here (delta handled separately)
* - blockBytes NOT updated here (delta=4 handled by incrementBlockBytes)
- *
+ *
* @param bytesWritten Number of bytes in the I/O operation
* @param overwrite Whether this is an overwrite operation
*/
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
index e667161559a..24201daa7e7 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.container.common.impl;
+import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_STORAGE_TYPE;
import static org.apache.hadoop.ozone.OzoneConsts.REPLICA_INDEX;
import static
org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData.KEYVALUE_YAML_TAG;
@@ -31,6 +32,8 @@
import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
import
org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
@@ -70,12 +73,18 @@ private ContainerDataYaml() {
/**
* Creates a .container file in yaml format.
*/
- public static void createContainerFile(ContainerData containerData, File
containerFile) throws IOException {
- // Create Yaml for given container type
- final Yaml yaml =
getYamlForContainerType(containerData.getContainerType(),
containerData.getReplicaIndex() > 0);
+ public static void createContainerFile(ContainerData containerData,
+ File containerFile) throws IOException {
+ boolean withReplicaIndex =
+ containerData instanceof KeyValueContainerData
+ && ((KeyValueContainerData) containerData).getReplicaIndex() > 0;
+ StorageType storageType =
+ containerData instanceof KeyValueContainerData
+ ? containerData.getStorageType() : null;
+ Yaml yaml = getYamlForContainerType(containerData.getContainerType(),
+ withReplicaIndex, storageType);
// Compute Checksum and update ContainerData
- containerData.computeAndSetContainerFileChecksum(yaml);
-
+ containerData.computeAndSetContainerFileChecksum();
// Write the ContainerData with checksum to Yaml file.
YamlUtils.dump(yaml, containerData, containerFile, LOG);
}
@@ -145,17 +154,8 @@ public static ContainerData readContainer(InputStream
input)
return containerData;
}
- /**
- * Given a ContainerType this method returns a Yaml representation of
- * the container properties.
- *
- * @param containerType type of container
- * @param withReplicaIndex in the container yaml
- * @return Yamal representation of container properties
- * @throws StorageContainerException if the type is unrecognized
- */
public static Yaml getYamlForContainerType(ContainerType containerType,
- boolean withReplicaIndex)
+ boolean withReplicaIndex, StorageType storageType)
throws StorageContainerException {
PropertyUtils propertyUtils = new PropertyUtils();
propertyUtils.setBeanAccess(BeanAccess.FIELD);
@@ -168,6 +168,10 @@ public static Yaml getYamlForContainerType(ContainerType
containerType,
yamlFields = new ArrayList<>(yamlFields);
yamlFields.add(REPLICA_INDEX);
}
+ if (storageType != null) {
+ yamlFields = new ArrayList<>(yamlFields);
+ yamlFields.add(CONTAINER_STORAGE_TYPE);
+ }
Representer representer = new ContainerDataRepresenter(yamlFields);
representer.setPropertyUtils(propertyUtils);
representer.addClassTag(
@@ -284,6 +288,12 @@ public Object construct(Node node) {
kvData.setReplicaIndex(
((Long) replicaIndex).intValue());
}
+ String storageTypeString = (String) nodes.get(CONTAINER_STORAGE_TYPE);
+ StorageType storageType = StringUtils.isEmpty(storageTypeString) ?
+ null : StorageType.valueOf(storageTypeString);
+ if (storageType != null) {
+ kvData.setStorageType(storageType);
+ }
return kvData;
}
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
index 34643cc6161..a32cdce53a1 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
@@ -20,6 +20,7 @@
import java.io.IOException;
import net.jcip.annotations.Immutable;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.StorageTypeProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.MetadataStorageReportProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
@@ -114,12 +115,12 @@ public StorageType getStorageType() {
return storageType;
}
- private StorageTypeProto getStorageTypeProto() throws IOException {
+ private StorageTypeProto getStorageTypeProto() throws
IllegalArgumentException {
return getStorageTypeProto(getStorageType());
}
public static StorageTypeProto getStorageTypeProto(StorageType type)
- throws IOException {
+ throws IllegalArgumentException {
StorageTypeProto storageTypeProto;
switch (type) {
case SSD:
@@ -138,12 +139,12 @@ public static StorageTypeProto
getStorageTypeProto(StorageType type)
storageTypeProto = StorageTypeProto.RAM_DISK;
break;
default:
- throw new IOException("Illegal Storage Type specified");
+ throw new IllegalArgumentException("Illegal Storage Type specified");
}
return storageTypeProto;
}
- public long getReserved() {
+ public long getReserved() {
return reserved;
}
@@ -155,8 +156,8 @@ public long getFsAvailable() {
return fsAvailable;
}
- private static StorageType getStorageType(StorageTypeProto proto) throws
- IOException {
+ public static StorageType getStorageType(StorageTypeProto proto) throws
+ IllegalArgumentException {
StorageType storageType;
switch (proto) {
case SSD:
@@ -175,7 +176,7 @@ private static StorageType getStorageType(StorageTypeProto
proto) throws
storageType = StorageType.RAM_DISK;
break;
default:
- throw new IOException("Illegal Storage Type specified");
+ throw new IllegalArgumentException("Illegal Storage Type specified");
}
return storageType;
}
@@ -184,9 +185,14 @@ private static StorageType getStorageType(StorageTypeProto
proto) throws
* Returns the StorageReportProto protoBuf message for the Storage Location
* report.
* @return StorageReportProto
- * @throws IOException In case, the storage type specified is invalid.
+ * @throws IllegalArgumentException In case, the storage type specified is
invalid.
*/
- public StorageReportProto getProtoBufMessage() throws IOException {
+ public StorageReportProto getProtoBufMessage() throws
IllegalArgumentException, IOException {
+ return getProtoBufMessage(null);
+ }
+
+ public StorageReportProto getProtoBufMessage(ConfigurationSource conf)
+ throws IOException {
StorageReportProto.Builder srb = StorageReportProto.newBuilder();
return srb.setStorageUuid(getId())
.setCapacity(getCapacity())
@@ -207,10 +213,10 @@ public StorageReportProto getProtoBufMessage() throws
IOException {
* Returns the MetadataStorageReportProto protoBuf message for the
* Storage Location report.
* @return MetadataStorageReportProto
- * @throws IOException In case, the storage type specified is invalid.
+ * @throws IllegalArgumentException In case, the storage type specified is
invalid.
*/
public MetadataStorageReportProto getMetadataProtoBufMessage()
- throws IOException {
+ throws IllegalArgumentException {
MetadataStorageReportProto.Builder srb =
MetadataStorageReportProto.newBuilder();
return srb.setCapacity(getCapacity())
@@ -226,11 +232,11 @@ public MetadataStorageReportProto
getMetadataProtoBufMessage()
* Returns the StorageLocationReport from the protoBuf message.
* @param report SCMStorageReport
* @return StorageLocationReport
- * @throws IOException in case of invalid storage type
+ * @throws IllegalArgumentException in case of invalid storage type
*/
public static StorageLocationReport getFromProtobuf(StorageReportProto
report)
- throws IOException {
+ throws IllegalArgumentException {
StorageLocationReport.Builder builder = StorageLocationReport.newBuilder();
builder.setId(report.getStorageUuid())
.setStorageLocation(report.getStorageLocation());
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java
index 3f6ba78e125..4ccd8538647 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java
@@ -56,7 +56,7 @@ protected long getReportFrequency() {
}
@Override
- protected NodeReportProto getReport() throws IOException {
+ protected NodeReportProto getReport() throws IOException,
IllegalArgumentException {
return getContext().getParent().getContainer().getNodeReport();
}
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java
index 03fa43fa841..beda2cb8ca0 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java
@@ -83,7 +83,7 @@ private void publishReport() {
} else {
context.refreshFullReport(report);
}
- } catch (IOException e) {
+ } catch (IOException | IllegalArgumentException e) {
LOG.error("Exception while publishing report.", e);
}
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index 27702064cf9..007b6ecba77 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -677,7 +677,7 @@ public ContainerController getController() {
* Returns node report of container storage usage.
*/
public StorageContainerDatanodeProtocolProtos.NodeReportProto getNodeReport()
- throws IOException {
+ throws IOException, IllegalArgumentException {
StorageLocationReport[] reports = volumeSet.getStorageReport();
StorageContainerDatanodeProtocolProtos.NodeReportProto.Builder nrb
= StorageContainerDatanodeProtocolProtos.
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
index 2e54a12ef8f..68e682f87ca 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
@@ -25,6 +25,7 @@
import java.util.UUID;
import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
@@ -94,8 +95,9 @@ public void testKeyValueData(ContainerTestVersionInfo
versionInfo) {
kvData.incrPendingDeletionBlocks(1, 256);
kvData.setSchemaVersion(
VersionedDatanodeFeatures.SchemaV3.chooseSchemaVersion(conf));
- long expectedDataHash = 1234L;
+ long expectedDataHash = 1234L;
kvData.setDataChecksum(expectedDataHash);
+ kvData.setStorageType(StorageType.SSD);
assertEquals(state, kvData.getState());
assertEquals(containerDBType, kvData.getContainerDBType());
@@ -110,12 +112,14 @@ public void testKeyValueData(ContainerTestVersionInfo
versionInfo) {
assertEquals(VersionedDatanodeFeatures.SchemaV3.chooseSchemaVersion(conf),
kvData.getSchemaVersion());
assertEquals(expectedDataHash, kvData.getDataChecksum());
+ assertEquals(StorageType.SSD, kvData.getStorageType());
KeyValueContainerData newKvData = new KeyValueContainerData(kvData);
assertEquals(kvData.getReplicaIndex(), newKvData.getReplicaIndex());
assertEquals(0, newKvData.getNumPendingDeletionBlocks());
assertEquals(0, newKvData.getDeleteTransactionId());
assertEquals(kvData.getSchemaVersion(), newKvData.getSchemaVersion());
+ assertEquals(kvData.getStorageType(), newKvData.getStorageType());
}
@ContainerTestVersionInfo.ContainerTest
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java
index a8723fc147c..b025803af7e 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java
@@ -70,7 +70,6 @@
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
-import org.yaml.snakeyaml.Yaml;
/**
* Tests processing of containers written with DB schema version 1,
@@ -597,10 +596,10 @@ private KeyValueContainerData newKvData() throws
IOException {
// Changing the paths above affects the checksum, so it was also removed
// from the container file and calculated at run time.
- Yaml yaml = ContainerDataYaml.getYamlForContainerType(
+ ContainerDataYaml.getYamlForContainerType(
kvData.getContainerType(),
- kvData.getReplicaIndex() > 0);
- kvData.computeAndSetContainerFileChecksum(yaml);
+ kvData.getReplicaIndex() > 0, kvData.getStorageType());
+ kvData.computeAndSetContainerFileChecksum();
KeyValueContainerUtil.parseKVContainerData(kvData, conf);
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
index 84a2672b3f9..ef77d276d8d 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
@@ -17,9 +17,11 @@
package org.apache.hadoop.ozone.container.common.impl;
+import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_STORAGE_TYPE;
import static
org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion.FILE_PER_CHUNK;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -32,6 +34,7 @@
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -41,6 +44,8 @@
import org.apache.hadoop.ozone.container.keyvalue.ContainerLayoutTestInfo;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
+import org.junit.jupiter.api.Test;
+import org.yaml.snakeyaml.Yaml;
/**
* This class tests create/read .container files.
@@ -68,9 +73,10 @@ private void setLayoutVersion(ContainerLayoutVersion
layoutVersion) {
* Creates a .container file. cleanup() should be called at the end of the
* test when container file is created.
*/
- private File createContainerFile(long containerID, int replicaIndex)
+ private File createContainerFile(long containerID, int replicaIndex,
StorageType storageType)
throws IOException {
- assertTrue(new File(testRoot).mkdirs());
+ File root = new File(testRoot);
+ assertTrue(root.mkdirs() || root.exists());
String containerPath = containerID + ".container";
@@ -78,6 +84,7 @@ private File createContainerFile(long containerID, int
replicaIndex)
containerID, layoutVersion, MAXSIZE,
UUID.randomUUID().toString(),
UUID.randomUUID().toString());
+ keyValueContainerData.setStorageType(storageType);
keyValueContainerData.setContainerDBType(CONTAINER_DB_TYPE);
keyValueContainerData.setMetadataPath(testRoot);
keyValueContainerData.setChunksPath(testRoot);
@@ -108,11 +115,12 @@ public void
testCreateContainerFile(ContainerLayoutVersion layout)
setLayoutVersion(layout);
long containerID = testContainerID++;
- File containerFile = createContainerFile(containerID, 7);
+ File containerFile = createContainerFile(containerID, 7, StorageType.SSD);
// Read from .container file, and verify data.
KeyValueContainerData kvData = (KeyValueContainerData) ContainerDataYaml
.readContainerFile(containerFile);
+ assertEquals(StorageType.SSD, kvData.getStorageType());
assertEquals(containerID, kvData.getContainerID());
assertEquals(ContainerProtos.ContainerType.KeyValueContainer, kvData
.getContainerType());
@@ -176,7 +184,7 @@ public void testCreateContainerFileWithoutReplicaIndex(
setLayoutVersion(layout);
long containerID = testContainerID++;
- File containerFile = createContainerFile(containerID, 0);
+ File containerFile = createContainerFile(containerID, 0, null);
final String content =
FileUtils.readFileToString(containerFile, Charset.defaultCharset());
@@ -241,7 +249,7 @@ public void
testChecksumInContainerFile(ContainerLayoutVersion layout) throws IO
setLayoutVersion(layout);
long containerID = testContainerID++;
- File containerFile = createContainerFile(containerID, 0);
+ File containerFile = createContainerFile(containerID, 0, null);
// Read from .container file, and verify data.
KeyValueContainerData kvData = (KeyValueContainerData)
ContainerDataYaml.readContainerFile(containerFile);
@@ -259,7 +267,7 @@ public void
testDataChecksumNotInContainerFile(ContainerLayoutVersion layout) th
setLayoutVersion(layout);
long containerID = testContainerID++;
- File containerFile = createContainerFile(containerID, 0);
+ File containerFile = createContainerFile(containerID, 0, null);
// Read from .container file. The kvData object should not have a data
hash because it was not persisted in this
// file.
@@ -278,7 +286,7 @@ public void testChecksumInContainerFileWithReplicaIndex(
setLayoutVersion(layout);
long containerID = testContainerID++;
- File containerFile = createContainerFile(containerID, 10);
+ File containerFile = createContainerFile(containerID, 10, null);
// Read from .container file, and verify data.
KeyValueContainerData kvData = (KeyValueContainerData) ContainerDataYaml
@@ -288,6 +296,44 @@ public void testChecksumInContainerFileWithReplicaIndex(
cleanup();
}
+ @Test
+ public void testChecksumCanBeVerifiedAfterRollbackWithStorageType()
+ throws IOException {
+ setLayoutVersion(FILE_PER_CHUNK);
+ long containerID = testContainerID++;
+
+ File containerFile = createContainerFile(containerID, 7, StorageType.SSD);
+
+ final String content =
+ FileUtils.readFileToString(containerFile, Charset.defaultCharset());
+ assertThat(content).contains(CONTAINER_STORAGE_TYPE);
+
+ KeyValueContainerData kvData = (KeyValueContainerData) ContainerDataYaml
+ .readContainerFile(containerFile);
+ String storedChecksum = kvData.getContainerFileChecksum();
+
+ assertEquals(computeContainerFileChecksum(kvData, null), storedChecksum);
+ assertThat(computeContainerFileChecksum(kvData, kvData.getStorageType()))
+ .isNotEqualTo(storedChecksum);
+ ContainerUtils.verifyContainerFileChecksum(kvData, conf);
+
+ cleanup();
+ }
+
+ private String computeContainerFileChecksum(KeyValueContainerData kvData,
+ StorageType storageType) throws IOException {
+ String storedChecksum = kvData.getContainerFileChecksum();
+ try {
+ kvData.setContainerFileChecksum(ContainerData.ZERO_CHECKSUM);
+ Yaml yaml = ContainerDataYaml.getYamlForContainerType(
+ kvData.getContainerType(), kvData.getReplicaIndex() > 0,
+ storageType);
+ return ContainerUtils.getContainerFileChecksum(yaml.dump(kvData));
+ } finally {
+ kvData.setContainerFileChecksum(storedChecksum);
+ }
+ }
+
private KeyValueContainerData getKeyValueContainerData() throws IOException {
String containerFile = "incorrect.checksum.container";
//Get file from resources folder
@@ -322,4 +368,19 @@ public void testDisabledChecksum(ContainerLayoutVersion
layout)
HDDS_CONTAINER_CHECKSUM_VERIFICATION_ENABLED, false);
ContainerUtils.verifyContainerFileChecksum(kvData, conf);
}
+
+ @Test
+ public void testCreateContainerFileWithoutStorageType() throws IOException {
+ setLayoutVersion(FILE_PER_CHUNK);
+ long containerID = testContainerID++;
+
+ File containerFile = createContainerFile(containerID, 0, null);
+
+ final String content =
+ FileUtils.readFileToString(containerFile, Charset.defaultCharset());
+
+ assertFalse(content.contains(CONTAINER_STORAGE_TYPE),
+ "StorageType shouldn't be persisted if it is null");
+ cleanup();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
