This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-14496-zdu
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-14496-zdu by this push:
new 020f8aa7b1b HDDS-14570. SCM should finalize all versions with one
Ratis request (#9715)
020f8aa7b1b is described below
commit 020f8aa7b1baa08d6ee4ebcb3eefa0571b1f3042
Author: Stephen O'Donnell <[email protected]>
AuthorDate: Thu Mar 19 21:31:55 2026 +0000
HDDS-14570. SCM should finalize all versions with one Ratis request (#9715)
---
.../ozone/upgrade/BasicUpgradeFinalizer.java | 7 +++++
.../DefaultUpgradeFinalizationExecutor.java | 4 +--
.../server/upgrade/FinalizationStateManager.java | 2 +-
.../upgrade/FinalizationStateManagerImpl.java | 19 +++++++++----
.../scm/server/upgrade/SCMUpgradeFinalizer.java | 29 ++++++++++++++------
.../hdds/scm/upgrade/TestScmFinalization.java | 32 ++++++----------------
6 files changed, 51 insertions(+), 42 deletions(-)
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java
index 4d75e88cdf8..fda69ff872f 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java
@@ -246,6 +246,13 @@ private static boolean isFinalized(Status status) {
public abstract void finalizeLayoutFeature(LayoutFeature lf, T context)
throws UpgradeException;
+ public void finalizeLayoutFeatures(Iterable<LayoutFeature> features, T
context)
+ throws UpgradeException {
+ for (LayoutFeature lf : features) {
+ finalizeLayoutFeature(lf, context);
+ }
+ }
+
protected void finalizeLayoutFeature(LayoutFeature lf, Optional<?
extends UpgradeAction> action, Storage storage)
throws UpgradeException {
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/DefaultUpgradeFinalizationExecutor.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/DefaultUpgradeFinalizationExecutor.java
index dad1ebd0d7e..fe689ae1517 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/DefaultUpgradeFinalizationExecutor.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/DefaultUpgradeFinalizationExecutor.java
@@ -66,8 +66,6 @@ public void execute(T component, BasicUpgradeFinalizer<T, ?>
finalizer)
protected void finalizeFeatures(T component,
BasicUpgradeFinalizer<T, ?> finalizer, Iterable<LayoutFeature> lfs)
throws UpgradeException {
- for (LayoutFeature lf: lfs) {
- finalizer.finalizeLayoutFeature(lf, component);
- }
+ finalizer.finalizeLayoutFeatures(lfs, component);
}
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java
index 6d9a61eb9b4..84cb6389ea6 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java
@@ -33,7 +33,7 @@ public interface FinalizationStateManager {
void removeFinalizingMark() throws IOException;
@Replicate
- void finalizeLayoutFeature(Integer layoutVersion)
+ void finalizeLayoutFeatures(Integer toLayoutVersion)
throws IOException;
/**
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java
index 4dde4089f0a..5ffa8062a5e 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java
@@ -99,8 +99,11 @@ public void addFinalizingMark() throws IOException {
}
@Override
- public void finalizeLayoutFeature(Integer layoutVersion) throws IOException {
- finalizeLayoutFeatureLocal(layoutVersion);
+ public void finalizeLayoutFeatures(Integer toVersion) throws IOException {
+ int startLayoutVersion = versionManager.getMetadataLayoutVersion() + 1;
+ for (int version = startLayoutVersion; version <= toVersion; version++) {
+ finalizeLayoutFeatureLocal(version);
+ }
}
/**
@@ -115,9 +118,15 @@ private void finalizeLayoutFeatureLocal(Integer
layoutVersion)
// version. This is updated in the replicated finalization steps.
// Layout version will be written to the DB as well so followers can
// finalize from a snapshot.
- HDDSLayoutFeature feature =
- (HDDSLayoutFeature)versionManager.getFeature(layoutVersion);
- upgradeFinalizer.replicatedFinalizationSteps(feature, upgradeContext);
+ if (versionManager.getMetadataLayoutVersion() >= layoutVersion) {
+ LOG.warn("Attempting to finalize layout feature for layout version {},
but " +
+ "current metadata layout version is {}. Skipping finalization for
this layout version.",
+ layoutVersion, versionManager.getMetadataLayoutVersion());
+ } else {
+ HDDSLayoutFeature feature =
+ (HDDSLayoutFeature) versionManager.getFeature(layoutVersion);
+ upgradeFinalizer.replicatedFinalizationSteps(feature, upgradeContext);
+ }
} finally {
checkpointLock.writeLock().unlock();
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java
index 553d1d09b19..773ade68a2f 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java
@@ -70,22 +70,33 @@ public void
preFinalizeUpgrade(SCMUpgradeFinalizationContext context)
@Override
public void finalizeLayoutFeature(LayoutFeature lf,
- SCMUpgradeFinalizationContext context) throws UpgradeException {
- // Run upgrade actions, update VERSION file, and update layout version in
- // DB.
+ SCMUpgradeFinalizationContext context) throws UpgradeException {
+ throw new UnsupportedOperationException("FinalizeLayoutFeature is not
supported in this implemementation. " +
+ "Please use finalizeLayoutFeatures instead.");
+ }
+
+ @Override
+ public void finalizeLayoutFeatures(Iterable<LayoutFeature> features,
SCMUpgradeFinalizationContext context)
+ throws UpgradeException {
+ int lastLv = -1;
+ for (LayoutFeature lf : features) {
+ lastLv = lf.layoutVersion();
+ }
try {
- context.getFinalizationStateManager()
- .finalizeLayoutFeature(lf.layoutVersion());
+ if (lastLv > -1) {
+ // If there are no feature to finalize we just skip this and the
finalize operation is a noop.
+ context.getFinalizationStateManager().finalizeLayoutFeatures(lastLv);
+ } else {
+ LOG.info("No layout features to finalize.");
+ }
} catch (IOException ex) {
- throw new UpgradeException(ex,
- UpgradeException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED);
+ throw new UpgradeException(ex,
UpgradeException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED);
}
}
/**
* Run on each SCM (leader and follower) when a layout feature is being
- * finalized to run its finalization actions, update the VERSION file, and
- * move the state of its in memory datanodes to healthy readonly.
+ * finalized to run its finalization actions, update the VERSION file.
*
* @param lf The layout feature that is being finalized.
* @param context Supplier of objects needed to run the steps.
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java
index a6bc7e4acaa..9364b9ff37c 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java
@@ -114,33 +114,17 @@ public void testUpgradeStateToCheckpointMapping() throws
Exception {
.build();
stateManager.setUpgradeContext(context);
- assertCurrentCheckpoint(scmContext, stateManager,
- FinalizationCheckpoint.FINALIZATION_REQUIRED);
+ assertCurrentCheckpoint(scmContext, stateManager,
FinalizationCheckpoint.FINALIZATION_REQUIRED);
stateManager.addFinalizingMark();
- assertCurrentCheckpoint(scmContext, stateManager,
- FinalizationCheckpoint.FINALIZATION_STARTED);
-
- for (HDDSLayoutFeature feature: HDDSLayoutFeature.values()) {
- // Cannot finalize initial version since we are already there.
- if (!feature.equals(HDDSLayoutFeature.INITIAL_VERSION)) {
- stateManager.finalizeLayoutFeature(feature.layoutVersion());
- if (versionManager.needsFinalization()) {
- assertCurrentCheckpoint(scmContext, stateManager,
- FinalizationCheckpoint.FINALIZATION_STARTED);
- } else {
- assertCurrentCheckpoint(scmContext, stateManager,
- FinalizationCheckpoint.MLV_EQUALS_SLV);
- }
- }
- }
- // Make sure we reached this checkpoint if we finished finalizing all
- // layout features.
- assertCurrentCheckpoint(scmContext, stateManager,
- FinalizationCheckpoint.MLV_EQUALS_SLV);
+ assertCurrentCheckpoint(scmContext, stateManager,
FinalizationCheckpoint.FINALIZATION_STARTED);
+ HDDSLayoutFeature[] finalizationFeatures = HDDSLayoutFeature.values();
+ HDDSLayoutFeature finalVersion =
finalizationFeatures[finalizationFeatures.length - 1];
+ assertCurrentCheckpoint(scmContext, stateManager,
FinalizationCheckpoint.FINALIZATION_STARTED);
+ stateManager.finalizeLayoutFeatures(finalVersion.layoutVersion());
+ assertCurrentCheckpoint(scmContext, stateManager,
FinalizationCheckpoint.MLV_EQUALS_SLV);
stateManager.removeFinalizingMark();
- assertCurrentCheckpoint(scmContext, stateManager,
- FinalizationCheckpoint.FINALIZATION_COMPLETE);
+ assertCurrentCheckpoint(scmContext, stateManager,
FinalizationCheckpoint.FINALIZATION_COMPLETE);
}
private void assertCurrentCheckpoint(SCMContext context,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
