This is an automated email from the ASF dual-hosted git repository.
erose pushed a commit to branch HDDS-14496-zdu
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-14496-zdu by this push:
new 50f407e2f61 HDDS-14732. Create a new VersionManager for unified
component versioning (#9897)
50f407e2f61 is described below
commit 50f407e2f61ad0d9ca959fe4bd9d8911ea61f330
Author: Ethan Rose <[email protected]>
AuthorDate: Thu Mar 26 08:56:23 2026 -0400
HDDS-14732. Create a new VersionManager for unified component versioning
(#9897)
---
.../annotations/RegisterValidatorProcessor.java | 2 +-
.../org/apache/hadoop/hdds/ComponentVersion.java | 49 ++++++-
.../java/org/apache/hadoop/hdds/HDDSVersion.java | 47 ++++---
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 42 +++++-
.../org/apache/hadoop/ozone/ClientVersion.java | 34 ++---
.../apache/hadoop/ozone/OzoneManagerVersion.java | 45 +++---
.../apache/hadoop/ozone/upgrade/LayoutFeature.java | 8 --
.../hadoop/hdds/AbstractComponentVersionTest.java | 135 ++++++++++++++++++
.../hadoop/hdds/ComponentVersionTestUtils.java | 51 +++++++
.../org/apache/hadoop/hdds/TestClientVersion.java} | 30 ++--
.../hdds/TestComponentVersionInvariants.java | 148 --------------------
.../org/apache/hadoop/hdds/TestHDDSVersion.java | 63 +++++++++
.../hadoop/hdds/TestOzoneManagerVersion.java | 64 +++++++++
.../hadoop/hdds/upgrade/TestHDDSLayoutFeature.java | 109 +++++++++++++++
.../hadoop/hdds/upgrade/HDDSVersionManager.java | 48 +++++++
.../ozone/upgrade/ComponentVersionManager.java | 151 +++++++++++++++++++++
.../upgrade/ComponentVersionManagerMetrics.java | 76 +++++++++++
.../hdds/upgrade/TestHDDSVersionManager.java | 69 ++++++++++
.../AbstractComponentVersionManagerTest.java | 149 ++++++++++++++++++++
.../upgrade/TestAbstractLayoutVersionManager.java | 7 +
.../ozone/upgrade/TestUpgradeFinalizerActions.java | 7 +
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 42 +++++-
.../hadoop/ozone/om/upgrade/OMVersionManager.java | 48 +++++++
.../ozone/om/upgrade/TestOMLayoutFeature.java | 109 +++++++++++++++
...anager.java => TestOMLayoutVersionManager.java} | 22 +--
.../ozone/om/upgrade/TestOMVersionManager.java | 140 +++++--------------
26 files changed, 1335 insertions(+), 360 deletions(-)
diff --git
a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java
b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java
index 6b26043efb2..c5639e11e54 100644
---
a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java
+++
b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java
@@ -54,7 +54,7 @@
public class RegisterValidatorProcessor extends AbstractProcessor {
public static final String ANNOTATION_SIMPLE_NAME = "RegisterValidator";
- public static final String VERSION_CLASS_NAME =
"org.apache.hadoop.hdds.ComponentVersion";
+ public static final String VERSION_CLASS_NAME =
"org.apache.hadoop.ozone.Version";
public static final String REQUEST_PROCESSING_PHASE_CLASS_NAME =
"org.apache.hadoop.ozone.om.request.validation" +
".RequestProcessingPhase";
public static final String APPLY_BEFORE_METHOD_NAME = "applyBefore";
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java
index 10f232ee0c9..9c1223f35c3 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java
@@ -21,26 +21,61 @@
import org.apache.hadoop.ozone.upgrade.UpgradeAction;
/**
- * Base type for component version enums.
+ * The logical versioning system used to track incompatible changes to a
component, regardless whether they affect disk
+ * or network compatibility between the same or different types of components.
+ *
+ * This interface is the base type for component version enums.
*/
public interface ComponentVersion {
/**
- * @return The serialized representation of this version. This is an opaque
value which should not be checked or
- * compared directly.
+ * Returns an integer representation of this version. To callers outside
this class, this is an opaque value which
+ * should not be checked or compared directly. {@link #isSupportedBy} should
be used for version comparisons.
+ *
+ * To implementors of this interface, versions should serialize such that
+ * {@code version1 <= version2} if and only if
+ * {@code version1.serialize() <= version2.serialize()}.
+ * Negative numbers may be used as serialized values to represent unknown
future versions which are trivially larger
+ * than all other versions.
+ *
+ * @return The serialized representation of this version.
*/
int serialize();
/**
- * @return the description of the version enum value.
+ * @return The description of this version.
*/
String description();
/**
- * Deserializes a ComponentVersion and checks if its feature set is
supported by the current ComponentVersion.
+ * @return The next version immediately following this one, or null if there
is no such version.
+ */
+ ComponentVersion nextVersion();
+
+ /**
+ * Uses the serialized representation of a ComponentVersion to check if its
feature set is supported by the current
+ * ComponentVersion.
*
- * @return true if this version supports the features of otherVersion. False
otherwise.
+ * @return true if this version supports the features of the provided
version. False otherwise.
*/
- boolean isSupportedBy(int serializedVersion);
+ default boolean isSupportedBy(int serializedVersion) {
+ if (serialize() < 0) {
+ // Our version is an unknown future version, it is not supported by any
other version.
+ return false;
+ } else if (serializedVersion < 0) {
+ // The other version is an unknown future version, it trivially supports
all other versions.
+ return true;
+ } else {
+ // If both versions have positive values, they represent concrete
versions and we can compare them directly.
+ return serialize() <= serializedVersion;
+ }
+ }
+
+ /**
+ * @return true if this version supports the features of the provided
version. False otherwise.
+ */
+ default boolean isSupportedBy(ComponentVersion other) {
+ return isSupportedBy(other.serialize());
+ }
default Optional<? extends UpgradeAction> action() {
return Optional.empty();
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java
index 6b4131e2226..51d4229a748 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java
@@ -21,13 +21,16 @@
import static java.util.stream.Collectors.toMap;
import java.util.Arrays;
-import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
/**
* Versioning for datanode.
*/
public enum HDDSVersion implements ComponentVersion {
+ ////////////////////////////// //////////////////////////////
+
DEFAULT_VERSION(0, "Initial version"),
SEPARATE_RATIS_PORTS_AVAILABLE(1, "Version with separated Ratis port."),
@@ -36,14 +39,18 @@ public enum HDDSVersion implements ComponentVersion {
STREAM_BLOCK_SUPPORT(3,
"This version has support for reading a block by streaming chunks."),
+ ZDU(100, "Version that supports zero downtime upgrade"),
+
FUTURE_VERSION(-1, "Used internally in the client when the server side is "
+ " newer and an unknown server version has arrived to the client.");
- public static final HDDSVersion SOFTWARE_VERSION = latest();
+ ////////////////////////////// //////////////////////////////
- private static final Map<Integer, HDDSVersion> BY_VALUE =
+ private static final SortedMap<Integer, HDDSVersion> BY_VALUE =
Arrays.stream(values())
- .collect(toMap(HDDSVersion::serialize, identity()));
+ .collect(toMap(HDDSVersion::serialize, identity(), (v1, v2) -> v1,
TreeMap::new));
+
+ public static final HDDSVersion SOFTWARE_VERSION =
BY_VALUE.get(BY_VALUE.lastKey());
private final int version;
private final String description;
@@ -58,31 +65,35 @@ public String description() {
return description;
}
+ /**
+ * @return The next version immediately following this one and excluding
FUTURE_VERSION,
+ * or null if there is no such version.
+ */
+ @Override
+ public HDDSVersion nextVersion() {
+ int nextOrdinal = ordinal() + 1;
+ if (nextOrdinal >= values().length - 1) {
+ return null;
+ }
+ return values()[nextOrdinal];
+ }
+
@Override
public int serialize() {
return version;
}
+ /**
+ * @param value The serialized version to convert.
+ * @return The version corresponding to this serialized value, or {@link
#FUTURE_VERSION} if no matching version is
+ * found.
+ */
public static HDDSVersion deserialize(int value) {
return BY_VALUE.getOrDefault(value, FUTURE_VERSION);
}
- @Override
- public boolean isSupportedBy(int serializedVersion) {
- // In order for the other serialized version to support this version's
features,
- // the other version must be equal or larger to this version.
- return deserialize(serializedVersion).compareTo(this) >= 0;
- }
-
@Override
public String toString() {
return name() + " (" + serialize() + ")";
}
-
- private static HDDSVersion latest() {
- HDDSVersion[] versions = HDDSVersion.values();
- // The last entry in the array will be `FUTURE_VERSION`. We want the entry
prior to this which defines the latest
- // version in the software.
- return versions[versions.length - 2];
- }
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java
index e78e1dcbffe..4e753c5cab3 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java
@@ -17,11 +17,21 @@
package org.apache.hadoop.hdds.upgrade;
+import static java.util.function.Function.identity;
+import static java.util.stream.Collectors.toMap;
+
+import java.util.Arrays;
import java.util.Optional;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.hdds.HDDSVersion;
import org.apache.hadoop.ozone.upgrade.LayoutFeature;
/**
- * List of HDDS Features.
+ * List of HDDS Layout Features. All version management has been migrated to
{@link HDDSVersion} and no new additions
+ * should be made to this class. Existing versions are kept here for backwards
compatibility when upgrading to this
+ * version from older versions.
*/
public enum HDDSLayoutFeature implements LayoutFeature {
////////////////////////////// //////////////////////////////
@@ -44,8 +54,14 @@ public enum HDDSLayoutFeature implements LayoutFeature {
WITNESSED_CONTAINER_DB_PROTO_VALUE(9, "ContainerID table schema to use value
type as proto"),
STORAGE_SPACE_DISTRIBUTION(10, "Enhanced block deletion function for storage
space distribution feature.");
+ // ALL NEW VERSIONS SHOULD NOW BE ADDED TO HDDSVersion
+
////////////////////////////// //////////////////////////////
+ private static final SortedMap<Integer, HDDSLayoutFeature> BY_VALUE =
+ Arrays.stream(values())
+ .collect(toMap(HDDSLayoutFeature::serialize, identity(), (v1, v2) ->
v1, TreeMap::new));
+
private final int layoutVersion;
private final String description;
private HDDSUpgradeAction scmAction;
@@ -90,6 +106,30 @@ public String description() {
return description;
}
+ /**
+ * @return The next version immediately following this one. If there is no
next version found in this enum,
+ * the next version is {@link HDDSVersion#ZDU}, since all HDDS versioning
has been migrated to
+ * {@link HDDSVersion} as part of the ZDU feature.
+ */
+ @Override
+ public ComponentVersion nextVersion() {
+ HDDSLayoutFeature nextFeature = BY_VALUE.get(layoutVersion + 1);
+ if (nextFeature == null) {
+ return HDDSVersion.ZDU;
+ } else {
+ return nextFeature;
+ }
+ }
+
+ /**
+ * @param version The serialized version to convert.
+ * @return The version corresponding to this serialized value, or {@code
null} if no matching version is
+ * found.
+ */
+ public static HDDSLayoutFeature deserialize(int version) {
+ return BY_VALUE.get(version);
+ }
+
@Override
public String toString() {
return name() + " (" + serialize() + ")";
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java
index ac20b5fe3b7..dd451ab052c 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java
@@ -21,7 +21,8 @@
import static java.util.stream.Collectors.toMap;
import java.util.Arrays;
-import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
import org.apache.hadoop.hdds.ComponentVersion;
/**
@@ -44,11 +45,11 @@ public enum ClientVersion implements ComponentVersion {
FUTURE_VERSION(-1, "Used internally when the server side is older and an"
+ " unknown client version has arrived from the client.");
- public static final ClientVersion CURRENT = latest();
-
- private static final Map<Integer, ClientVersion> BY_VALUE =
+ private static final SortedMap<Integer, ClientVersion> BY_VALUE =
Arrays.stream(values())
- .collect(toMap(ClientVersion::serialize, identity()));
+ .collect(toMap(ClientVersion::serialize, identity(), (v1, v2) -> v1,
TreeMap::new));
+
+ public static final ClientVersion CURRENT = BY_VALUE.get(BY_VALUE.lastKey());
private final int version;
private final String description;
@@ -63,6 +64,15 @@ public String description() {
return description;
}
+ @Override
+ public ClientVersion nextVersion() {
+ int nextOrdinal = ordinal() + 1;
+ if (nextOrdinal >= values().length - 1) {
+ return null;
+ }
+ return values()[nextOrdinal];
+ }
+
@Override
public int serialize() {
return version;
@@ -72,22 +82,8 @@ public static ClientVersion deserialize(int value) {
return BY_VALUE.getOrDefault(value, FUTURE_VERSION);
}
- @Override
- public boolean isSupportedBy(int serializedVersion) {
- // In order for the other serialized version to support this version's
features,
- // the other version must be equal or larger to this version.
- return deserialize(serializedVersion).compareTo(this) >= 0;
- }
-
@Override
public String toString() {
return name() + " (" + serialize() + ")";
}
-
- private static ClientVersion latest() {
- ClientVersion[] versions = ClientVersion.values();
- // The last entry in the array will be `FUTURE_VERSION`. We want the entry
prior to this which defines the latest
- // version in the software.
- return versions[versions.length - 2];
- }
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
index 9ec8870855e..163696b4f15 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
@@ -21,13 +21,17 @@
import static java.util.stream.Collectors.toMap;
import java.util.Arrays;
-import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
import org.apache.hadoop.hdds.ComponentVersion;
/**
* Versioning for Ozone Manager.
*/
public enum OzoneManagerVersion implements ComponentVersion {
+
+ ////////////////////////////// //////////////////////////////
+
DEFAULT_VERSION(0, "Initial version"),
S3G_PERSISTENT_CONNECTIONS(1,
"New S3G persistent connection support is present in OM."),
@@ -54,15 +58,19 @@ public enum OzoneManagerVersion implements ComponentVersion
{
S3_LIST_MULTIPART_UPLOADS_PAGINATION(11,
"OzoneManager version that supports S3 list multipart uploads API with
pagination"),
-
+
+ ZDU(100, "OzoneManager version that supports zero downtime upgrade"),
+
FUTURE_VERSION(-1, "Used internally in the client when the server side is "
+ " newer and an unknown server version has arrived to the client.");
- public static final OzoneManagerVersion SOFTWARE_VERSION = latest();
+ ////////////////////////////// //////////////////////////////
- private static final Map<Integer, OzoneManagerVersion> BY_VALUE =
+ private static final SortedMap<Integer, OzoneManagerVersion> BY_VALUE =
Arrays.stream(values())
- .collect(toMap(OzoneManagerVersion::serialize, identity()));
+ .collect(toMap(OzoneManagerVersion::serialize, identity(), (v1, v2)
-> v1, TreeMap::new));
+
+ public static final OzoneManagerVersion SOFTWARE_VERSION =
BY_VALUE.get(BY_VALUE.lastKey());
private final int version;
private final String description;
@@ -82,26 +90,31 @@ public int serialize() {
return version;
}
+ /**
+ * @param value The serialized version to convert.
+ * @return The version corresponding to this serialized value, or {@link
#FUTURE_VERSION} if no matching version is
+ * found.
+ */
public static OzoneManagerVersion deserialize(int value) {
return BY_VALUE.getOrDefault(value, FUTURE_VERSION);
}
+
+ /**
+ * @return The next version immediately following this one and excluding
FUTURE_VERSION,
+ * or null if there is no such version.
+ */
@Override
- public boolean isSupportedBy(int serializedVersion) {
- // In order for the other serialized version to support this version's
features,
- // the other version must be equal or larger to this version.
- return deserialize(serializedVersion).compareTo(this) >= 0;
+ public OzoneManagerVersion nextVersion() {
+ int nextOrdinal = ordinal() + 1;
+ if (nextOrdinal >= values().length - 1) {
+ return null;
+ }
+ return values()[nextOrdinal];
}
@Override
public String toString() {
return name() + " (" + serialize() + ")";
}
-
- private static OzoneManagerVersion latest() {
- OzoneManagerVersion[] versions = OzoneManagerVersion.values();
- // The last entry in the array will be `FUTURE_VERSION`. We want the entry
prior to this which defines the latest
- // version in the software.
- return versions[versions.length - 2];
- }
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java
index 848a35104e5..7f7374c8137 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java
@@ -29,12 +29,4 @@ public interface LayoutFeature extends ComponentVersion {
default int serialize() {
return this.layoutVersion();
}
-
- @Override
- default boolean isSupportedBy(int serializedVersion) {
- // In order for the other serialized version to support this version's
features,
- // the other version must be equal or larger to this version.
- // We can compare the values directly since there is no FUTURE_VERSION for
layout features.
- return serializedVersion >= layoutVersion();
- }
}
diff --git
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/AbstractComponentVersionTest.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/AbstractComponentVersionTest.java
new file mode 100644
index 00000000000..c9b2a32647f
--- /dev/null
+++
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/AbstractComponentVersionTest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds;
+
+import static
org.apache.hadoop.hdds.ComponentVersionTestUtils.assertNotSupportedBy;
+import static
org.apache.hadoop.hdds.ComponentVersionTestUtils.assertSupportedBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+
+/**
+ * Shared invariants for component version enums.
+ */
+public abstract class AbstractComponentVersionTest {
+
+ protected abstract ComponentVersion[] getValues();
+
+ protected abstract ComponentVersion getDefaultVersion();
+
+ protected abstract ComponentVersion getFutureVersion();
+
+ protected abstract ComponentVersion deserialize(int value);
+
+ // FUTURE_VERSION is the latest
+ @Test
+ public void testFutureVersionHasTheHighestOrdinal() {
+ ComponentVersion[] values = getValues();
+ ComponentVersion futureValue = getFutureVersion();
+ assertEquals(values[values.length - 1], futureValue);
+ }
+
+ // FUTURE_VERSION's internal version id is -1
+ @Test
+ public void testFutureVersionSerializesToMinusOne() {
+ ComponentVersion futureValue = getFutureVersion();
+ assertEquals(-1, futureValue.serialize());
+ }
+
+ // DEFAULT_VERSION's internal version id is 0
+ @Test
+ public void testDefaultVersionSerializesToZero() {
+ ComponentVersion defaultValue = getDefaultVersion();
+ assertEquals(0, defaultValue.serialize());
+ }
+
+ // known (non-future) versions are strictly increasing
+ @Test
+ public void testSerializedValuesAreMonotonic() {
+ ComponentVersion[] values = getValues();
+ int knownVersionCount = values.length - 1;
+ for (int i = 1; i < knownVersionCount; i++) {
+ assertTrue(values[i].serialize() > values[i - 1].serialize(),
+ "Expected known version serialization to increase: " + values[i - 1]
+ " -> " + values[i]);
+ }
+ }
+
+ @Test
+ public void testNextVersionProgression() {
+ ComponentVersion[] values = getValues();
+ ComponentVersion futureValue = getFutureVersion();
+ int knownVersionCount = values.length - 1;
+ for (int i = 0; i < knownVersionCount - 1; i++) {
+ assertEquals(values[i + 1], values[i].nextVersion(),
+ "Expected nextVersion progression for " + values[i]);
+ }
+ assertNull(values[knownVersionCount - 1].nextVersion(),
+ "Expected latest known version to have no nextVersion");
+ assertNull(futureValue.nextVersion(),
+ "Expected FUTURE_VERSION.nextVersion() to return null");
+ }
+
+ @Test
+ public void testOnlyEqualOrHigherVersionsCanSupportAFeature() {
+ ComponentVersion[] values = getValues();
+ int knownVersionCount = values.length - 1;
+ for (int featureIndex = 0; featureIndex < knownVersionCount;
featureIndex++) {
+ ComponentVersion requiredFeature = values[featureIndex];
+ for (int providerIndex = 0; providerIndex < knownVersionCount;
providerIndex++) {
+ ComponentVersion provider = values[providerIndex];
+ if (providerIndex >= featureIndex) {
+ assertSupportedBy(requiredFeature, provider);
+ } else {
+ assertNotSupportedBy(requiredFeature, provider);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testFutureVersionSupportsAllKnownVersions() {
+ ComponentVersion[] values = getValues();
+ int unknownFutureVersion = Integer.MAX_VALUE;
+ for (ComponentVersion knownVersion : values) {
+ if (knownVersion == getFutureVersion()) {
+ // FUTURE_VERSION with serialized value < 0 is considered larger than
any version with a concrete
+ // positive value.
+ assertFalse(knownVersion.isSupportedBy(unknownFutureVersion),
knownVersion +
+ " should not support unknown future version " +
unknownFutureVersion);
+ } else {
+ assertTrue(knownVersion.isSupportedBy(unknownFutureVersion),
knownVersion +
+ " should support unknown future version " + unknownFutureVersion);
+ }
+ }
+ }
+
+ @Test
+ public void testVersionSerDes() {
+ for (ComponentVersion version : getValues()) {
+ assertEquals(version, deserialize(version.serialize()));
+ }
+ }
+
+ @Test
+ public void testDeserializeUnknownReturnsFutureVersion() {
+ assertEquals(getFutureVersion(), deserialize(Integer.MAX_VALUE));
+ }
+}
diff --git
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ComponentVersionTestUtils.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ComponentVersionTestUtils.java
new file mode 100644
index 00000000000..0531523cfac
--- /dev/null
+++
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ComponentVersionTestUtils.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Shared assertions for {@link ComponentVersion} tests.
+ */
+public final class ComponentVersionTestUtils {
+
+ private ComponentVersionTestUtils() { }
+
+ public static void assertSupportedBy(
+ ComponentVersion requiredFeature, ComponentVersion provider) {
+ assertSupportedBy(requiredFeature, provider, true);
+ }
+
+ public static void assertNotSupportedBy(
+ ComponentVersion requiredFeature, ComponentVersion provider) {
+ assertSupportedBy(requiredFeature, provider, false);
+ }
+
+ /**
+ * Helper method to test support by passing both serialized and deserialized
versions.
+ */
+ private static void assertSupportedBy(
+ ComponentVersion requiredFeature, ComponentVersion provider, boolean
expected) {
+ int providerSerializedVersion = provider.serialize();
+ assertEquals(expected,
requiredFeature.isSupportedBy(providerSerializedVersion),
+ "Expected support check via serialized overload to match for version "
+ + providerSerializedVersion);
+ assertEquals(expected, requiredFeature.isSupportedBy(provider),
+ "Expected support check via version overload to match for " +
provider);
+ }
+}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestClientVersion.java
similarity index 57%
copy from
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java
copy to
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestClientVersion.java
index 848a35104e5..721fb2466c9 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java
+++
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestClientVersion.java
@@ -15,26 +15,32 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.upgrade;
+package org.apache.hadoop.hdds;
-import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.ozone.ClientVersion;
/**
- * Generic Layout feature interface for Ozone.
+ * Invariants for {@link ClientVersion}.
*/
-public interface LayoutFeature extends ComponentVersion {
- int layoutVersion();
+public class TestClientVersion extends AbstractComponentVersionTest {
@Override
- default int serialize() {
- return this.layoutVersion();
+ protected ComponentVersion[] getValues() {
+ return ClientVersion.values();
}
@Override
- default boolean isSupportedBy(int serializedVersion) {
- // In order for the other serialized version to support this version's
features,
- // the other version must be equal or larger to this version.
- // We can compare the values directly since there is no FUTURE_VERSION for
layout features.
- return serializedVersion >= layoutVersion();
+ protected ComponentVersion getDefaultVersion() {
+ return ClientVersion.DEFAULT_VERSION;
+ }
+
+ @Override
+ protected ComponentVersion getFutureVersion() {
+ return ClientVersion.FUTURE_VERSION;
+ }
+
+ @Override
+ protected ComponentVersion deserialize(int value) {
+ return ClientVersion.deserialize(value);
}
}
diff --git
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestComponentVersionInvariants.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestComponentVersionInvariants.java
deleted file mode 100644
index 0edab7e76d2..00000000000
---
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestComponentVersionInvariants.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.params.provider.Arguments.arguments;
-
-import java.util.stream.Stream;
-import org.apache.hadoop.ozone.ClientVersion;
-import org.apache.hadoop.ozone.OzoneManagerVersion;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
-import org.junit.jupiter.params.provider.MethodSource;
-
-/**
- * Test to ensure Component version instances conform with invariants relied
- * upon in other parts of the codebase.
- */
-public class TestComponentVersionInvariants {
-
- public static Stream<Arguments> values() {
- return Stream.of(
- arguments(
- HDDSVersion.values(),
- HDDSVersion.DEFAULT_VERSION,
- HDDSVersion.FUTURE_VERSION),
- arguments(
- ClientVersion.values(),
- ClientVersion.DEFAULT_VERSION,
- ClientVersion.FUTURE_VERSION),
- arguments(
- OzoneManagerVersion.values(),
- OzoneManagerVersion.DEFAULT_VERSION,
- OzoneManagerVersion.FUTURE_VERSION)
- );
- }
-
- // FUTURE_VERSION is the latest
- @ParameterizedTest
- @MethodSource("values")
- public void testFutureVersionHasTheHighestOrdinal(ComponentVersion[] values,
ComponentVersion defaultValue,
- ComponentVersion futureValue) {
-
- assertEquals(values[values.length - 1], futureValue);
- }
-
- // FUTURE_VERSION's internal version id is -1
- @ParameterizedTest
- @MethodSource("values")
- public void testFutureVersionSerializesToMinusOne(ComponentVersion[] values,
ComponentVersion defaultValue,
- ComponentVersion futureValue) {
- assertEquals(-1, futureValue.serialize());
-
- }
-
- // DEFAULT_VERSION's internal version id is 0
- @ParameterizedTest
- @MethodSource("values")
- public void testDefaultVersionSerializesToZero(ComponentVersion[] values,
ComponentVersion defaultValue,
- ComponentVersion futureValue) {
- assertEquals(0, defaultValue.serialize());
- }
-
- // versions are increasing monotonically by one
- @ParameterizedTest
- @MethodSource("values")
- public void testSerializedValuesAreMonotonic(ComponentVersion[] values,
ComponentVersion defaultValue,
- ComponentVersion futureValue) {
- int startValue = defaultValue.serialize();
- // we skip the future version at the last position
- for (int i = 0; i < values.length - 1; i++) {
- assertEquals(values[i].serialize(), startValue++);
- }
- assertEquals(values.length, ++startValue);
- }
-
- @ParameterizedTest
- @MethodSource("values")
- public void testVersionIsSupportedByItself(ComponentVersion[] values,
ComponentVersion defaultValue,
- ComponentVersion futureValue) {
- for (ComponentVersion value : values) {
- assertTrue(value.isSupportedBy(value.serialize()));
- }
- }
-
- @ParameterizedTest
- @MethodSource("values")
- public void
testOnlyEqualOrHigherVersionsCanSupportAFeature(ComponentVersion[] values,
ComponentVersion defaultValue,
- ComponentVersion futureValue) {
- int knownVersionCount = values.length - 1;
- for (int featureIndex = 0; featureIndex < knownVersionCount;
featureIndex++) {
- ComponentVersion requiredFeature = values[featureIndex];
- for (int providerIndex = 0; providerIndex < knownVersionCount;
providerIndex++) {
- ComponentVersion provider = values[providerIndex];
- boolean expected = providerIndex >= featureIndex;
- assertEquals(expected,
requiredFeature.isSupportedBy(provider.serialize()));
- }
- }
- }
-
- @ParameterizedTest
- @MethodSource("values")
- public void testFutureVersionSupportsAllKnownVersions(ComponentVersion[]
values, ComponentVersion defaultValue,
- ComponentVersion futureValue) {
- int unknownFutureVersion = Integer.MAX_VALUE;
- for (ComponentVersion requiredFeature : values) {
- assertTrue(requiredFeature.isSupportedBy(unknownFutureVersion));
- }
- }
-
- @Test
- public void testHDDSVersionSerDes() {
- for (HDDSVersion version: HDDSVersion.values()) {
- assertEquals(version, HDDSVersion.deserialize(version.serialize()));
- }
- }
-
- @Test
- public void testOMVersionSerDes() {
- for (OzoneManagerVersion version: OzoneManagerVersion.values()) {
- assertEquals(version,
OzoneManagerVersion.deserialize(version.serialize()));
- }
- }
-
- @Test
- public void testClientVersionSerDes() {
- for (ClientVersion version: ClientVersion.values()) {
- assertEquals(version, ClientVersion.deserialize(version.serialize()));
- }
- }
-}
diff --git
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java
new file mode 100644
index 00000000000..89d24d3c82e
--- /dev/null
+++
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+
+/**
+ * Invariants for {@link HDDSVersion}.
+ */
+public class TestHDDSVersion extends AbstractComponentVersionTest {
+
+ @Override
+ protected ComponentVersion[] getValues() {
+ return HDDSVersion.values();
+ }
+
+ @Override
+ protected ComponentVersion getDefaultVersion() {
+ return HDDSVersion.DEFAULT_VERSION;
+ }
+
+ @Override
+ protected ComponentVersion getFutureVersion() {
+ return HDDSVersion.FUTURE_VERSION;
+ }
+
+ @Override
+ protected ComponentVersion deserialize(int value) {
+ return HDDSVersion.deserialize(value);
+ }
+
+ @Test
+ public void testKnownVersionNumbersAreContiguousExceptForZDU() {
+ HDDSVersion[] values = HDDSVersion.values();
+ int knownVersionCount = values.length - 1;
+ for (int i = 0; i < knownVersionCount - 1; i++) {
+ HDDSVersion current = values[i];
+ HDDSVersion next = values[i + 1];
+ if (next == HDDSVersion.ZDU) {
+ assertEquals(100, next.serialize());
+ } else {
+ assertEquals(current.serialize() + 1, next.serialize());
+ }
+ }
+ }
+}
diff --git
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java
new file mode 100644
index 00000000000..bb94ac938fd
--- /dev/null
+++
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.apache.hadoop.ozone.OzoneManagerVersion;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Invariants for {@link OzoneManagerVersion}.
+ */
+public class TestOzoneManagerVersion extends AbstractComponentVersionTest {
+
+ @Override
+ protected ComponentVersion[] getValues() {
+ return OzoneManagerVersion.values();
+ }
+
+ @Override
+ protected ComponentVersion getDefaultVersion() {
+ return OzoneManagerVersion.DEFAULT_VERSION;
+ }
+
+ @Override
+ protected ComponentVersion getFutureVersion() {
+ return OzoneManagerVersion.FUTURE_VERSION;
+ }
+
+ @Override
+ protected ComponentVersion deserialize(int value) {
+ return OzoneManagerVersion.deserialize(value);
+ }
+
+ @Test
+ public void testKnownVersionNumbersAreContiguousExceptForZDU() {
+ OzoneManagerVersion[] values = OzoneManagerVersion.values();
+ int knownVersionCount = values.length - 1;
+ for (int i = 0; i < knownVersionCount - 1; i++) {
+ OzoneManagerVersion current = values[i];
+ OzoneManagerVersion next = values[i + 1];
+ if (next == OzoneManagerVersion.ZDU) {
+ assertEquals(100, next.serialize());
+ } else {
+ assertEquals(current.serialize() + 1, next.serialize());
+ }
+ }
+ }
+}
diff --git
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java
new file mode 100644
index 00000000000..fbb89220e97
--- /dev/null
+++
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.upgrade;
+
+import static
org.apache.hadoop.hdds.ComponentVersionTestUtils.assertNotSupportedBy;
+import static
org.apache.hadoop.hdds.ComponentVersionTestUtils.assertSupportedBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.hadoop.hdds.HDDSVersion;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests invariants for legacy HDDS layout feature versions.
+ */
+public class TestHDDSLayoutFeature {
+ @Test
+ public void testHDDSLayoutFeaturesHaveIncreasingLayoutVersion() {
+ HDDSLayoutFeature[] values = HDDSLayoutFeature.values();
+ int currVersion = -1;
+ for (HDDSLayoutFeature lf : values) {
+ // This will skip the jump from the last HDDSLayoutFeature to
HDDSVersion#ZDU,
+ // since that is expected to be a larger version increment.
+ assertEquals(currVersion + 1, lf.layoutVersion(),
+ "Expected monotonically increasing layout version for " + lf);
+ currVersion = lf.layoutVersion();
+ }
+ }
+
+ /**
+ * All incompatible changes to HDDS (SCM and Datanodes) should now be added
to {@link HDDSVersion}.
+ */
+ @Test
+ public void testNoNewHDDSLayoutFeaturesAdded() {
+ int numHDDSLayoutFeatures = HDDSLayoutFeature.values().length;
+ HDDSLayoutFeature lastFeature =
HDDSLayoutFeature.values()[numHDDSLayoutFeatures - 1];
+ assertEquals(11, numHDDSLayoutFeatures);
+ assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION, lastFeature);
+ assertEquals(10, lastFeature.layoutVersion());
+ }
+
+ @Test
+ public void testNextVersion() {
+ HDDSLayoutFeature[] values = HDDSLayoutFeature.values();
+ for (int i = 1; i < values.length; i++) {
+ HDDSLayoutFeature previous = values[i - 1];
+ HDDSLayoutFeature current = values[i];
+ assertEquals(current, previous.nextVersion(),
+ "Expected " + previous + ".nextVersion() to be " + current);
+ }
+ // The last layout feature should point us to the ZDU version to switch to
using HDDSVersion.
+ assertEquals(HDDSVersion.ZDU, values[values.length - 1].nextVersion());
+ }
+
+ @Test
+ public void testSerDes() {
+ for (HDDSLayoutFeature version : HDDSLayoutFeature.values()) {
+ assertEquals(version,
HDDSLayoutFeature.deserialize(version.serialize()));
+ }
+ }
+
+ @Test
+ public void testDeserializeUnknownVersionReturnsNull() {
+ assertNull(HDDSLayoutFeature.deserialize(-1));
+ assertNull(HDDSLayoutFeature.deserialize(Integer.MAX_VALUE));
+ // HDDSLayoutFeature can only deserialize values from its own enum.
+ assertNull(HDDSLayoutFeature.deserialize(HDDSVersion.ZDU.serialize()));
+ }
+
+ @Test
+ public void testIsSupportedByFeatureBoundary() {
+ for (HDDSLayoutFeature feature : HDDSLayoutFeature.values()) {
+ // A layout feature should support itself.
+ int layoutVersion = feature.layoutVersion();
+ assertSupportedBy(feature, feature);
+ if (layoutVersion > 0) {
+ // A layout feature should not be supported by older features.
+ HDDSLayoutFeature previousFeature =
HDDSLayoutFeature.values()[layoutVersion - 1];
+ assertNotSupportedBy(feature, previousFeature);
+ }
+ }
+ }
+
+ @Test
+ public void testAllLayoutFeaturesAreSupportedByFutureVersions() {
+ for (HDDSLayoutFeature feature : HDDSLayoutFeature.values()) {
+ assertSupportedBy(feature, HDDSVersion.ZDU);
+ assertSupportedBy(feature, HDDSVersion.FUTURE_VERSION);
+ // No ComponentVersion instance represents an arbitrary future version.
+ assertTrue(feature.isSupportedBy(Integer.MAX_VALUE));
+ }
+ }
+}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java
new file mode 100644
index 00000000000..ad6bad1c702
--- /dev/null
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.upgrade;
+
+import java.io.IOException;
+import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.hdds.HDDSVersion;
+import org.apache.hadoop.ozone.upgrade.ComponentVersionManager;
+
+/**
+ * Component version manager for HDDS (Datanodes and SCM).
+ */
+public class HDDSVersionManager extends ComponentVersionManager {
+ public HDDSVersionManager(int serializedApparentVersion) throws IOException {
+ super(computeApparentVersion(serializedApparentVersion),
HDDSVersion.SOFTWARE_VERSION);
+ }
+
+ /**
+ * If the apparent version stored on the disk is >= 100, it indicates the
component has been finalized for the
+ * ZDU feature, and the apparent version corresponds to a version in {@link
HDDSVersion}.
+ * If the apparent version stored on the disk is < 100, it indicates the
component is not yet finalized for the
+ * ZDU feature, and the apparent version corresponds to a version in {@link
HDDSLayoutFeature}.
+ */
+ private static ComponentVersion computeApparentVersion(int
serializedApparentVersion) {
+ if (serializedApparentVersion < HDDSVersion.ZDU.serialize()) {
+ return HDDSLayoutFeature.deserialize(serializedApparentVersion);
+ } else {
+ return HDDSVersion.deserialize(serializedApparentVersion);
+ }
+ }
+
+ // TODO HDDS-14826: Register upgrade actions based on annotations
+}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java
new file mode 100644
index 00000000000..11bf4a3f228
--- /dev/null
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.upgrade;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import org.apache.hadoop.hdds.ComponentVersion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Tracks information about the apparent version, software version, and
finalization status of a component.
+ *
+ * Software version: The {@link ComponentVersion} of the code that is
currently running.
+ * This is always the highest component version within the code and does
not change while the process is running.
+ *
+ * Apparent version: The {@link ComponentVersion} the software is acting as,
which is persisted to the disk.
+ * The apparent version determines the API that is exposed by the component
and the format it uses to persist data.
+ * Using an apparent version less than software version allows us to
support rolling upgrades and downgrades.
+ *
+ * Pre-finalized: State a component enters when the apparent version on disk
is less than the software version.
+ * At this time all other machines may or may not be running the new bits,
new features are blocked, and downgrade
+ * is allowed.
+ *
+ * Finalized: State a component enters when the apparent version is equal to
the software version.
+ * A component transitions from pre-finalized to finalized when it receives
a finalize command from the
+ * admin. At this time all machines are running the new bits, and even
though this component is finalized,
+ * different types of components may not be. Downgrade is not allowed after
this point.
+ *
+ */
+public abstract class ComponentVersionManager implements Closeable {
+ // Apparent version may be updated during the finalization process.
+ private volatile ComponentVersion apparentVersion;
+ // Software version will never change.
+ private final ComponentVersion softwareVersion;
+ private final ComponentVersionManagerMetrics metrics;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ComponentVersionManager.class);
+
+ protected ComponentVersionManager(ComponentVersion apparentVersion,
ComponentVersion softwareVersion)
+ throws IOException {
+ this.apparentVersion = apparentVersion;
+ this.softwareVersion = softwareVersion;
+
+ if (!apparentVersion.isSupportedBy(softwareVersion)) {
+ throw new IOException(
+ "Cannot initialize ComponentVersionManager. Apparent version "
+ + apparentVersion + " is larger than software version "
+ + softwareVersion);
+ }
+
+ LOG.info("Initializing version manager with apparent version {} and
software version {}",
+ apparentVersion, softwareVersion);
+ this.metrics = ComponentVersionManagerMetrics.create(this);
+ }
+
+ public ComponentVersion getApparentVersion() {
+ return apparentVersion;
+ }
+
+ public ComponentVersion getSoftwareVersion() {
+ return softwareVersion;
+ }
+
+ public boolean isAllowed(ComponentVersion version) {
+ return version.isSupportedBy(apparentVersion);
+ }
+
+ public boolean needsFinalization() {
+ return !apparentVersion.equals(softwareVersion);
+ }
+
+ /**
+ * @return An Iterable of all versions after the current apparent version
which still need to be finalized. If this
+ * component is already finalized, the Iterable will be empty.
+ */
+ public Iterable<ComponentVersion> getUnfinalizedVersions() {
+ return () -> new Iterator<ComponentVersion>() {
+ private ComponentVersion currentVersion = apparentVersion;
+
+ @Override
+ public boolean hasNext() {
+ return currentVersion.nextVersion() != null;
+ }
+
+ @Override
+ public ComponentVersion next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ currentVersion = currentVersion.nextVersion();
+ return currentVersion;
+ }
+ };
+ }
+
+ /**
+ * Validates that the provided version is valid to finalize to, and if so,
updates the in-memory apparent version to
+ * this version. Also logs corresponding messages about finalization status.
+ *
+ * @param newApparentVersion The version to mark as finalized.
+ */
+ public void markFinalized(ComponentVersion newApparentVersion) {
+ String versionMsg = "Software version: " + softwareVersion
+ + ", apparent version: " + apparentVersion
+ + ", provided version: " + newApparentVersion
+ + ".";
+
+ if (newApparentVersion.isSupportedBy(apparentVersion)) {
+ LOG.info("Finalize attempt on a version which has already been
finalized. {} This can happen when " +
+ "Raft Log is replayed during service restart.", versionMsg);
+ } else {
+ ComponentVersion nextVersion = apparentVersion.nextVersion();
+ if (nextVersion == null) {
+ throw new IllegalArgumentException("Attempt to finalize when no future
versions exist." + versionMsg);
+ } else if (nextVersion.equals(newApparentVersion)) {
+ apparentVersion = newApparentVersion;
+ LOG.info("Version {} has been finalized.", apparentVersion);
+ if (!needsFinalization()) {
+ LOG.info("Finalization is complete.");
+ }
+ } else {
+ throw new IllegalArgumentException(
+ "Finalize attempt on a version that is newer than the next feature
to be finalized. " + versionMsg);
+ }
+ }
+ }
+
+ @Override
+ public void close() {
+ metrics.unRegister();
+ }
+}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManagerMetrics.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManagerMetrics.java
new file mode 100644
index 00000000000..3cafd99d48e
--- /dev/null
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManagerMetrics.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.upgrade;
+
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.Interns;
+import org.apache.hadoop.ozone.OzoneConsts;
+
+/**
+ * Metrics for {@link ComponentVersionManager}.
+ */
+@Metrics(about = "Component Version Manager Metrics", context =
OzoneConsts.OZONE)
+public final class ComponentVersionManagerMetrics implements MetricsSource {
+
+ public static final String METRICS_SOURCE_NAME =
+ ComponentVersionManagerMetrics.class.getSimpleName();
+
+ private static final MetricsInfo SOFTWARE_VERSION = Interns.info(
+ "SoftwareVersion",
+ "Software version in serialized int form.");
+ private static final MetricsInfo APPARENT_VERSION = Interns.info(
+ "ApparentVersion",
+ "Current apparent version in serialized int form.");
+
+ private final ComponentVersionManager versionManager;
+
+ private ComponentVersionManagerMetrics(ComponentVersionManager
versionManager) {
+ this.versionManager = versionManager;
+ }
+
+ public static ComponentVersionManagerMetrics create(ComponentVersionManager
versionManager) {
+ ComponentVersionManagerMetrics metrics = (ComponentVersionManagerMetrics)
DefaultMetricsSystem.instance()
+ .getSource(METRICS_SOURCE_NAME);
+ if (metrics == null) {
+ return DefaultMetricsSystem.instance().register(
+ METRICS_SOURCE_NAME,
+ "Metrics for component version management.",
+ new ComponentVersionManagerMetrics(versionManager));
+ }
+ return metrics;
+ }
+
+ @Override
+ public void getMetrics(MetricsCollector collector, boolean all) {
+ MetricsRecordBuilder builder = collector.addRecord(METRICS_SOURCE_NAME);
+ builder
+ .addGauge(SOFTWARE_VERSION,
+ versionManager.getSoftwareVersion().serialize())
+ .addGauge(APPARENT_VERSION,
+ versionManager.getApparentVersion().serialize());
+ }
+
+ public void unRegister() {
+ DefaultMetricsSystem.instance().unregisterSource(METRICS_SOURCE_NAME);
+ }
+}
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java
new file mode 100644
index 00000000000..1e76ff91be3
--- /dev/null
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.upgrade;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Stream;
+import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.hdds.HDDSVersion;
+import org.apache.hadoop.ozone.upgrade.AbstractComponentVersionManagerTest;
+import org.apache.hadoop.ozone.upgrade.ComponentVersionManager;
+import org.junit.jupiter.params.provider.Arguments;
+
+/**
+ * Tests for {@link HDDSVersionManager}.
+ */
+class TestHDDSVersionManager extends AbstractComponentVersionManagerTest {
+
+ private static final List<ComponentVersion> ALL_VERSIONS;
+
+ static {
+ ALL_VERSIONS = new ArrayList<>(Arrays.asList(HDDSLayoutFeature.values()));
+
+ for (HDDSVersion version : HDDSVersion.values()) {
+ // Add all defined versions after and including ZDU to get the complete
version list.
+ if (HDDSVersion.ZDU.isSupportedBy(version) && version !=
HDDSVersion.FUTURE_VERSION) {
+ ALL_VERSIONS.add(version);
+ }
+ }
+ }
+
+ public static Stream<Arguments> preFinalizedVersionArgs() {
+ return ALL_VERSIONS.stream()
+ .limit(ALL_VERSIONS.size() - 1)
+ .map(Arguments::of);
+ }
+
+ @Override
+ protected ComponentVersionManager createManager(int
serializedApparentVersion) throws IOException {
+ return new HDDSVersionManager(serializedApparentVersion);
+ }
+
+ @Override
+ protected List<ComponentVersion> allVersionsInOrder() {
+ return ALL_VERSIONS;
+ }
+
+ @Override
+ protected ComponentVersion expectedSoftwareVersion() {
+ return HDDSVersion.SOFTWARE_VERSION;
+ }
+}
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java
new file mode 100644
index 00000000000..0bf439326ca
--- /dev/null
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.upgrade;
+
+import static org.apache.ozone.test.MetricsAsserts.assertGauge;
+import static org.apache.ozone.test.MetricsAsserts.getMetrics;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.Mockito;
+
+/**
+ * Shared tests for concrete {@link ComponentVersionManager} implementations.
+ */
+public abstract class AbstractComponentVersionManagerTest {
+
+ protected abstract ComponentVersionManager createManager(int
serializedApparentVersion) throws IOException;
+
+ protected abstract List<ComponentVersion> allVersionsInOrder();
+
+ protected abstract ComponentVersion expectedSoftwareVersion();
+
+ @AfterEach
+ public void cleanupMetricsSource() {
+
DefaultMetricsSystem.instance().unregisterSource(ComponentVersionManagerMetrics.METRICS_SOURCE_NAME);
+ }
+
+ @Test
+ public void testApparentVersionTranslation() throws Exception {
+ for (ComponentVersion apparentVersion : allVersionsInOrder()) {
+ try (ComponentVersionManager versionManager =
createManager(apparentVersion.serialize())) {
+ assertApparentVersion(versionManager, apparentVersion);
+ }
+ }
+ }
+
+ @Test
+ public void testApparentVersionBehindSoftwareVersion() {
+ int serializedNextVersion = expectedSoftwareVersion().serialize() + 1;
+ assertThrows(IOException.class, () ->
createManager(serializedNextVersion));
+ }
+
+ @ParameterizedTest
+ // Child classes must implement this as a static method to provide the
versions to start finalization from.
+ @MethodSource("preFinalizedVersionArgs")
+ public void testFinalizationFromEarlierVersions(ComponentVersion
apparentVersion) throws Exception {
+ List<ComponentVersion> allVersions = allVersionsInOrder();
+ int apparentVersionIndex = allVersions.indexOf(apparentVersion);
+ assertTrue(apparentVersionIndex >= 0, "Apparent version " +
apparentVersion + " must exist");
+ Iterator<ComponentVersion> expectedVersions =
allVersions.subList(apparentVersionIndex + 1, allVersions.size())
+ .iterator();
+
+ try (ComponentVersionManager versionManager =
createManager(apparentVersion.serialize())) {
+ assertApparentVersion(versionManager, apparentVersion);
+
+ for (ComponentVersion versionToFinalize :
versionManager.getUnfinalizedVersions()) {
+ assertTrue(versionManager.needsFinalization());
+ assertFalse(versionManager.isAllowed(versionToFinalize),
+ "Unfinalized version " + versionToFinalize + " should not be
allowed by apparent version "
+ + versionManager.getApparentVersion());
+ assertTrue(expectedVersions.hasNext());
+ assertEquals(expectedVersions.next(), versionToFinalize);
+
+ versionManager.markFinalized(versionToFinalize);
+ assertApparentVersion(versionManager, versionToFinalize);
+ }
+
+ assertFalse(expectedVersions.hasNext());
+ assertThrows(NoSuchElementException.class, expectedVersions::next);
+ }
+ }
+
+ @Test
+ public void testFinalizationFromSoftwareVersionNoOp() throws Exception {
+ try (ComponentVersionManager versionManager =
createManager(expectedSoftwareVersion().serialize())) {
+ assertApparentVersion(versionManager, expectedSoftwareVersion());
+ assertFalse(versionManager.needsFinalization());
+
assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext());
+
+ versionManager.markFinalized(expectedSoftwareVersion());
+
+ assertApparentVersion(versionManager, expectedSoftwareVersion());
+ assertFalse(versionManager.needsFinalization());
+
assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext());
+ }
+ }
+
+ @Test
+ public void testFinalizationOfNonExistentVersion() throws Exception {
+ try (ComponentVersionManager versionManager =
createManager(expectedSoftwareVersion().serialize())) {
+ assertApparentVersion(versionManager, expectedSoftwareVersion());
+ assertFalse(versionManager.needsFinalization());
+
assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext());
+
+ ComponentVersion mockVersion = Mockito.mock(ComponentVersion.class);
+ when(mockVersion.isSupportedBy(any())).thenReturn(false);
+
+ assertThrows(IllegalArgumentException.class, () ->
versionManager.markFinalized(mockVersion));
+ // The failed finalization call should not have changed the version
manager's state.
+ assertApparentVersion(versionManager, expectedSoftwareVersion());
+ assertFalse(versionManager.needsFinalization());
+
assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext());
+ }
+ }
+
+ private void assertApparentVersion(ComponentVersionManager versionManager,
ComponentVersion apparentVersion) {
+ assertEquals(apparentVersion, versionManager.getApparentVersion());
+ assertTrue(versionManager.isAllowed(apparentVersion), apparentVersion + "
should be allowed");
+ assertEquals(expectedSoftwareVersion(),
versionManager.getSoftwareVersion(),
+ "Software version should never change");
+ if (!versionManager.needsFinalization()) {
+ assertTrue(versionManager.isAllowed(expectedSoftwareVersion()),
+ "Software version should always be allowed when finalized");
+ }
+ MetricsRecordBuilder metrics =
getMetrics(ComponentVersionManagerMetrics.METRICS_SOURCE_NAME);
+ assertGauge("SoftwareVersion", expectedSoftwareVersion().serialize(),
metrics);
+ assertGauge("ApparentVersion", apparentVersion.serialize(), metrics);
+ }
+}
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java
index f006698518a..6cd9391e28f 100644
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java
@@ -29,6 +29,7 @@
import java.util.Iterator;
import javax.management.MBeanServer;
import javax.management.ObjectName;
+import org.apache.hadoop.hdds.ComponentVersion;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -188,6 +189,12 @@ public int layoutVersion() {
public String description() {
return null;
}
+
+ @Override
+ public ComponentVersion nextVersion() {
+ // TODO HDDS-14826 will remove this test. No need to add handling
for this new method.
+ return null;
+ }
};
}
return lfs;
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java
index ade86b488b9..89a52b7c31d 100644
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java
@@ -86,6 +86,13 @@ public String description() {
return null;
}
+ @Override
+ public MockLayoutFeature nextVersion() {
+ // TODO HDDS-14826 will remove the tests that are using this. No need to
provide an implementation for this new
+ // method.
+ return null;
+ }
+
@Override
public String toString() {
return name() + " (" + serialize() + ")";
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
index c19b720682d..9f69215b694 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
@@ -17,11 +17,21 @@
package org.apache.hadoop.ozone.om.upgrade;
+import static java.util.function.Function.identity;
+import static java.util.stream.Collectors.toMap;
+
+import java.util.Arrays;
import java.util.Optional;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.ozone.OzoneManagerVersion;
import org.apache.hadoop.ozone.upgrade.LayoutFeature;
/**
- * List of OM Layout features / versions.
+ * List of OM Layout Features. All version management has been migrated to
{@link OzoneManagerVersion} and no new
+ * additions should be made to this class. Existing versions are kept here for
backwards compatibility when upgrading
+ * to this version from older versions.
*/
public enum OMLayoutFeature implements LayoutFeature {
////////////////////////////// //////////////////////////////
@@ -46,8 +56,14 @@ public enum OMLayoutFeature implements LayoutFeature {
DELEGATION_TOKEN_SYMMETRIC_SIGN(8, "Delegation token signed by symmetric
key"),
SNAPSHOT_DEFRAG(9, "Supporting defragmentation of snapshot");
+ // ALL NEW VERSIONS SHOULD NOW BE ADDED TO OzoneManagerVersion
+
/////////////////////////////// /////////////////////////////
+ private static final SortedMap<Integer, OMLayoutFeature> BY_VALUE =
+ Arrays.stream(values())
+ .collect(toMap(OMLayoutFeature::serialize, identity(), (v1, v2) ->
v1, TreeMap::new));
+
private final int layoutVersion;
private final String description;
private OmUpgradeAction action;
@@ -62,6 +78,15 @@ public int layoutVersion() {
return layoutVersion;
}
+ /**
+ * @param version The serialized version to convert.
+ * @return The version corresponding to this serialized value, or {@code
null} if no matching version is
+ * found.
+ */
+ public static OMLayoutFeature deserialize(int version) {
+ return BY_VALUE.get(version);
+ }
+
@Override
public String description() {
return description;
@@ -84,6 +109,21 @@ public void addAction(OmUpgradeAction upgradeAction) {
}
}
+ /**
+ * @return The next version immediately following this one. If there is no
next version found in this enum,
+ * the next version is {@link OzoneManagerVersion#ZDU}, since all OM
versioning has been migrated to
+ * {@link OzoneManagerVersion} as part of the ZDU feature.
+ */
+ @Override
+ public ComponentVersion nextVersion() {
+ OMLayoutFeature nextFeature = BY_VALUE.get(layoutVersion + 1);
+ if (nextFeature == null) {
+ return OzoneManagerVersion.ZDU;
+ } else {
+ return nextFeature;
+ }
+ }
+
@Override
public Optional<OmUpgradeAction> action() {
return Optional.ofNullable(action);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java
new file mode 100644
index 00000000000..f250928b2a9
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.upgrade;
+
+import java.io.IOException;
+import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.ozone.OzoneManagerVersion;
+import org.apache.hadoop.ozone.upgrade.ComponentVersionManager;
+
+/**
+ * Component version manager for Ozone Manager.
+ */
+public class OMVersionManager extends ComponentVersionManager {
+ public OMVersionManager(int serializedApparentVersion) throws IOException {
+ super(computeApparentVersion(serializedApparentVersion),
OzoneManagerVersion.SOFTWARE_VERSION);
+ }
+
+ /**
+ * If the apparent version stored on the disk is >= 100, it indicates the
component has been finalized for the
+ * ZDU feature, and the apparent version corresponds to a version in {@link
OzoneManagerVersion}.
+ * If the apparent version stored on the disk is < 100, it indicates the
component is not yet finalized for the
+ * ZDU feature, and the apparent version corresponds to a version in {@link
OMLayoutFeature}.
+ */
+ private static ComponentVersion computeApparentVersion(int
serializedApparentVersion) {
+ if (serializedApparentVersion < OzoneManagerVersion.ZDU.serialize()) {
+ return OMLayoutFeature.deserialize(serializedApparentVersion);
+ } else {
+ return OzoneManagerVersion.deserialize(serializedApparentVersion);
+ }
+ }
+
+ // TODO HDDS-14826: Register upgrade actions based on annotations
+}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java
new file mode 100644
index 00000000000..9e98935b5b3
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.upgrade;
+
+import static
org.apache.hadoop.hdds.ComponentVersionTestUtils.assertNotSupportedBy;
+import static
org.apache.hadoop.hdds.ComponentVersionTestUtils.assertSupportedBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.hadoop.ozone.OzoneManagerVersion;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests invariants for legacy OM layout feature versions.
+ */
+public class TestOMLayoutFeature {
+ @Test
+ public void testOMLayoutFeaturesHaveIncreasingLayoutVersion() {
+ OMLayoutFeature[] values = OMLayoutFeature.values();
+ int currVersion = -1;
+ for (OMLayoutFeature lf : values) {
+ // This will skip the jump from the last OMLayoutFeature to
OzoneManagerVersion#ZDU,
+ // since that is expected to be a larger version increment.
+ assertEquals(currVersion + 1, lf.layoutVersion(),
+ "Expected monotonically increasing layout version for " + lf);
+ currVersion = lf.layoutVersion();
+ }
+ }
+
+ /**
+ * All incompatible changes to OM should now be added to {@link
OzoneManagerVersion}.
+ */
+ @Test
+ public void testNoNewOMLayoutFeaturesAdded() {
+ int numOMLayoutFeatures = OMLayoutFeature.values().length;
+ OMLayoutFeature lastFeature = OMLayoutFeature.values()[numOMLayoutFeatures
- 1];
+ assertEquals(10, numOMLayoutFeatures);
+ assertEquals(OMLayoutFeature.SNAPSHOT_DEFRAG, lastFeature);
+ assertEquals(9, lastFeature.layoutVersion());
+ }
+
+ @Test
+ public void testNextVersion() {
+ OMLayoutFeature[] values = OMLayoutFeature.values();
+ for (int i = 1; i < values.length; i++) {
+ OMLayoutFeature previous = values[i - 1];
+ OMLayoutFeature current = values[i];
+ assertEquals(current, previous.nextVersion(),
+ "Expected " + previous + ".nextVersion() to be " + current);
+ }
+ // The last layout feature should point us to the ZDU version to switch to
using OzoneManagerVersion.
+ assertEquals(OzoneManagerVersion.ZDU, values[values.length -
1].nextVersion());
+ }
+
+ @Test
+ public void testSerDes() {
+ for (OMLayoutFeature version : OMLayoutFeature.values()) {
+ assertEquals(version, OMLayoutFeature.deserialize(version.serialize()));
+ }
+ }
+
+ @Test
+ public void testDeserializeUnknownVersionReturnsNull() {
+ assertNull(OMLayoutFeature.deserialize(-1));
+ assertNull(OMLayoutFeature.deserialize(Integer.MAX_VALUE));
+ // OMLayoutFeature can only deserialize values from its own enum.
+
assertNull(OMLayoutFeature.deserialize(OzoneManagerVersion.ZDU.serialize()));
+ }
+
+ @Test
+ public void testIsSupportedByFeatureBoundary() {
+ for (OMLayoutFeature feature : OMLayoutFeature.values()) {
+ // A layout feature should support itself.
+ int layoutVersion = feature.layoutVersion();
+ assertSupportedBy(feature, feature);
+ if (layoutVersion > 0) {
+ // A layout feature should not be supported by older features.
+ OMLayoutFeature previousFeature =
OMLayoutFeature.values()[layoutVersion - 1];
+ assertNotSupportedBy(feature, previousFeature);
+ }
+ }
+ }
+
+ @Test
+ public void testAllLayoutFeaturesAreSupportedByFutureVersions() {
+ for (OMLayoutFeature feature : OMLayoutFeature.values()) {
+ assertSupportedBy(feature, OzoneManagerVersion.ZDU);
+ assertSupportedBy(feature, OzoneManagerVersion.FUTURE_VERSION);
+ // No ComponentVersion instance represents an arbitrary unknown future
version.
+ assertTrue(feature.isSupportedBy(Integer.MAX_VALUE));
+ }
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutVersionManager.java
similarity index 87%
copy from
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java
copy to
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutVersionManager.java
index 388b5134c41..2c2044e9d04 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutVersionManager.java
@@ -44,7 +44,7 @@
/**
* Test OM layout version management.
*/
-public class TestOMVersionManager {
+public class TestOMLayoutVersionManager {
@Test
public void testOMLayoutVersionManager() throws IOException {
@@ -65,26 +65,6 @@ public void testOMLayoutVersionManagerInitError() {
assertEquals(NOT_SUPPORTED_OPERATION, ome.getResult());
}
- @Test
- public void testOMLayoutFeaturesHaveIncreasingLayoutVersion()
- throws Exception {
- OMLayoutFeature[] values = OMLayoutFeature.values();
- int currVersion = -1;
- OMLayoutFeature lastFeature = null;
- for (OMLayoutFeature lf : values) {
- assertEquals(currVersion + 1, lf.layoutVersion());
- currVersion = lf.layoutVersion();
- lastFeature = lf;
- }
- lastFeature.addAction(arg -> {
- String v = arg.getVersion();
- });
-
- OzoneManager omMock = mock(OzoneManager.class);
- lastFeature.action().get().execute(omMock);
- verify(omMock, times(1)).getVersion();
- }
-
@Test
/*
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java
index 388b5134c41..eed52733acb 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java
@@ -17,128 +17,52 @@
package org.apache.hadoop.ozone.om.upgrade;
-import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION;
-import static
org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.INITIAL_VERSION;
-import static
org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager.OM_UPGRADE_CLASS_PACKAGE;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.Mockito.anyString;
-import static org.mockito.Mockito.doCallRealMethod;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
import java.io.IOException;
-import java.lang.reflect.Method;
+import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Optional;
-import org.apache.hadoop.ozone.om.OzoneManager;
-import org.apache.hadoop.ozone.om.exceptions.OMException;
-import org.apache.hadoop.ozone.om.request.OMClientRequest;
-import org.junit.jupiter.api.Test;
+import java.util.List;
+import java.util.stream.Stream;
+import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.ozone.OzoneManagerVersion;
+import org.apache.hadoop.ozone.upgrade.AbstractComponentVersionManagerTest;
+import org.apache.hadoop.ozone.upgrade.ComponentVersionManager;
+import org.junit.jupiter.params.provider.Arguments;
/**
- * Test OM layout version management.
+ * Tests for {@link OMVersionManager}.
*/
-public class TestOMVersionManager {
-
- @Test
- public void testOMLayoutVersionManager() throws IOException {
- OMLayoutVersionManager omVersionManager =
- new OMLayoutVersionManager();
+class TestOMVersionManager extends AbstractComponentVersionManagerTest {
- // Initial Version is always allowed.
- assertTrue(omVersionManager.isAllowed(INITIAL_VERSION));
- assertThat(INITIAL_VERSION.layoutVersion())
- .isLessThanOrEqualTo(omVersionManager.getMetadataLayoutVersion());
- }
+ private static final List<ComponentVersion> ALL_VERSIONS;
- @Test
- public void testOMLayoutVersionManagerInitError() {
- int lV = OMLayoutFeature.values()[OMLayoutFeature.values().length - 1]
- .layoutVersion() + 1;
- OMException ome = assertThrows(OMException.class, () -> new
OMLayoutVersionManager(lV));
- assertEquals(NOT_SUPPORTED_OPERATION, ome.getResult());
- }
-
- @Test
- public void testOMLayoutFeaturesHaveIncreasingLayoutVersion()
- throws Exception {
- OMLayoutFeature[] values = OMLayoutFeature.values();
- int currVersion = -1;
- OMLayoutFeature lastFeature = null;
- for (OMLayoutFeature lf : values) {
- assertEquals(currVersion + 1, lf.layoutVersion());
- currVersion = lf.layoutVersion();
- lastFeature = lf;
+ static {
+ ALL_VERSIONS = new ArrayList<>(Arrays.asList(OMLayoutFeature.values()));
+ for (OzoneManagerVersion version : OzoneManagerVersion.values()) {
+ // Add all defined versions after and including ZDU to get the complete
version list.
+ if (OzoneManagerVersion.ZDU.isSupportedBy(version) && version !=
OzoneManagerVersion.FUTURE_VERSION) {
+ ALL_VERSIONS.add(version);
+ }
}
- lastFeature.addAction(arg -> {
- String v = arg.getVersion();
- });
-
- OzoneManager omMock = mock(OzoneManager.class);
- lastFeature.action().get().execute(omMock);
- verify(omMock, times(1)).getVersion();
}
- @Test
-
- /*
- * The OMLayoutFeatureAspect relies on the fact that the OM client
- * request handler class has a preExecute method with first argument as
- * 'OzoneManager'. If that is not true, please fix
- * OMLayoutFeatureAspect#beforeRequestApplyTxn.
- */
- public void testOmClientRequestPreExecuteIsCompatibleWithAspect() {
- Method[] methods = OMClientRequest.class.getMethods();
-
- Optional<Method> preExecuteMethod = Arrays.stream(methods)
- .filter(m -> m.getName().equals("preExecute"))
- .findFirst();
-
- assertTrue(preExecuteMethod.isPresent());
-
assertThat(preExecuteMethod.get().getParameterCount()).isGreaterThanOrEqualTo(1);
- assertEquals(OzoneManager.class,
- preExecuteMethod.get().getParameterTypes()[0]);
+ public static Stream<Arguments> preFinalizedVersionArgs() {
+ return ALL_VERSIONS.stream()
+ .limit(ALL_VERSIONS.size() - 1)
+ .map(Arguments::of);
}
- @Test
- public void testOmUpgradeActionsRegistered() throws Exception {
- OMLayoutVersionManager lvm = new OMLayoutVersionManager(); // MLV >= 0
- assertFalse(lvm.needsFinalization());
-
- // INITIAL_VERSION is finalized, hence should not register.
- Optional<OmUpgradeAction> action =
- INITIAL_VERSION.action();
- assertFalse(action.isPresent());
-
- lvm = mock(OMLayoutVersionManager.class);
- when(lvm.getMetadataLayoutVersion()).thenReturn(-1);
- doCallRealMethod().when(lvm).registerUpgradeActions(anyString());
- lvm.registerUpgradeActions(OM_UPGRADE_CLASS_PACKAGE);
-
- action = INITIAL_VERSION.action();
- assertTrue(action.isPresent());
- assertEquals(MockOmUpgradeAction.class, action.get().getClass());
- OzoneManager omMock = mock(OzoneManager.class);
- action.get().execute(omMock);
- verify(omMock, times(1)).getVersion();
+ @Override
+ protected ComponentVersionManager createManager(int
serializedApparentVersion) throws IOException {
+ return new OMVersionManager(serializedApparentVersion);
}
- /**
- * Mock OM upgrade action class.
- */
- @UpgradeActionOm(feature =
- INITIAL_VERSION)
- public static class MockOmUpgradeAction implements OmUpgradeAction {
+ @Override
+ protected List<ComponentVersion> allVersionsInOrder() {
+ return ALL_VERSIONS;
+ }
- @Override
- public void execute(OzoneManager arg) {
- arg.getVersion();
- }
+ @Override
+ protected ComponentVersion expectedSoftwareVersion() {
+ return OzoneManagerVersion.SOFTWARE_VERSION;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
