This is an automated email from the ASF dual-hosted git repository.
errose28 pushed a commit to branch HDDS-14496-zdu
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-14496-zdu by this push:
new 524b2370493 HDDS-15196. Split HDDS upgrade actions into specific SCM
and Datanode actions. (#10206)
524b2370493 is described below
commit 524b23704932d9e3b552b84f8c40611d0223c8f5
Author: Ethan Rose <[email protected]>
AuthorDate: Mon May 11 11:12:36 2026 -0400
HDDS-15196. Split HDDS upgrade actions into specific SCM and Datanode
actions. (#10206)
---
.../upgrade/ComponentUpgradeActionProvider.java | 2 +-
.../hdds/upgrade/DatanodeUpgradeAction.java} | 6 +-
.../upgrade/DatanodeUpgradeActionProvider.java} | 30 +++----
.../apache/hadoop/hdds/upgrade/package-info.java} | 9 +-
.../common/statemachine/DatanodeStateMachine.java | 3 +-
.../ContainerTableSchemaFinalizeAction.java | 9 +-
.../upgrade/DatanodeSchemaV2FinalizeAction.java | 9 +-
.../upgrade/DatanodeSchemaV3FinalizeAction.java | 9 +-
.../ScmHAFinalizeUpgradeActionDatanode.java | 9 +-
.../ozone/upgrade/UpgradeActionDatanode.java} | 17 +---
.../apache/hadoop/ozone/upgrade/package-info.java} | 9 +-
.../docs/content/design/upgrade-dev-primer.md | 2 +-
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 13 +--
.../hdds/upgrade/HDDSLayoutVersionManager.java | 79 ++++++------------
.../upgrade/AbstractUpgradeActionProvider.java | 95 ++++++++++++++++++++++
.../hadoop/hdds/upgrade/TestHDDSLayoutFeature.java | 0
.../hdds/upgrade/TestHDDSLayoutVersionManager.java | 79 +++++++++++++-----
.../hadoop/hdds/upgrade/test/MockComponent.java | 63 --------------
.../ozone/upgrade/TestUpgradeFinalizerActions.java | 48 +----------
.../hdds/scm/server/StorageContainerManager.java | 3 +-
.../ScmOnFinalizeActionForDatanodeSchemaV2.java | 9 +-
.../hadoop/hdds/upgrade/ScmUpgradeAction.java} | 6 +-
.../hdds/upgrade/ScmUpgradeActionProvider.java} | 30 +++----
.../apache/hadoop/hdds/upgrade/package-info.java} | 9 +-
.../hadoop/ozone/upgrade/UpgradeActionScm.java} | 17 +---
.../apache/hadoop/ozone/upgrade/package-info.java} | 9 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 8 +-
.../datanode/schemaupgrade/UpgradeUtils.java | 2 +-
.../ozone/om/upgrade/OMUpgradeActionProvider.java | 49 ++---------
.../scm/ReconStorageContainerManagerFacade.java | 2 +-
.../ozone/recon/scm/TestReconNodeManager.java | 2 +-
.../hadoop/ozone/freon/DatanodeSimulator.java | 2 +-
32 files changed, 281 insertions(+), 358 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentUpgradeActionProvider.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentUpgradeActionProvider.java
index bf2a8543d16..48bee3a6868 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentUpgradeActionProvider.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentUpgradeActionProvider.java
@@ -27,7 +27,7 @@
* @param <A> concrete upgrade action type (for example OM-specific or
HDDS-specific)
*/
@FunctionalInterface
-public interface ComponentUpgradeActionProvider<A> {
+public interface ComponentUpgradeActionProvider<A extends UpgradeAction<?>> {
/**
* Returns all upgrade actions from this provider, keyed by component
version.
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/upgrade/DatanodeUpgradeAction.java
similarity index 75%
copy from
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
copy to
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/upgrade/DatanodeUpgradeAction.java
index 81e1ade3700..f014c50b2fb 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/upgrade/DatanodeUpgradeAction.java
@@ -17,10 +17,12 @@
package org.apache.hadoop.hdds.upgrade;
+import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.upgrade.UpgradeAction;
/**
- * Upgrade Action for SCM and DataNodes.
+ * Datanode Upgrade Action interface. An upgrade action is an operation that
+ * needs to be executed during finalization.
*/
-public interface HDDSUpgradeAction<T> extends UpgradeAction<T> {
+public interface DatanodeUpgradeAction extends
UpgradeAction<DatanodeStateMachine> {
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentUpgradeActionProvider.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/upgrade/DatanodeUpgradeActionProvider.java
similarity index 50%
copy from
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentUpgradeActionProvider.java
copy to
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/upgrade/DatanodeUpgradeActionProvider.java
index bf2a8543d16..a2e9ca8a925 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentUpgradeActionProvider.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/upgrade/DatanodeUpgradeActionProvider.java
@@ -15,24 +15,26 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.upgrade;
+package org.apache.hadoop.hdds.upgrade;
-import java.util.Map;
import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.ozone.upgrade.AbstractUpgradeActionProvider;
+import org.apache.hadoop.ozone.upgrade.UpgradeActionDatanode;
/**
- * Supplies upgrade actions keyed by {@link ComponentVersion}. Implementations
typically perform classpath scanning or
- * return a fixed map for tests. The component version manager decides when
each action is invoked.
- *
- * @param <A> concrete upgrade action type (for example OM-specific or
HDDS-specific)
+ * Loads {@link DatanodeUpgradeAction} implementations annotated with {@link
UpgradeActionDatanode}.
*/
-@FunctionalInterface
-public interface ComponentUpgradeActionProvider<A> {
+public final class DatanodeUpgradeActionProvider extends
AbstractUpgradeActionProvider<DatanodeUpgradeAction> {
+
+ public static final String DATANODE_UPGRADE_CLASS_PACKAGE =
"org.apache.hadoop.ozone.container";
+
+ public DatanodeUpgradeActionProvider() {
+ super(UpgradeActionDatanode.class, DatanodeUpgradeAction.class,
DATANODE_UPGRADE_CLASS_PACKAGE);
+ }
- /**
- * Returns all upgrade actions from this provider, keyed by component
version.
- * <p>
- * Implementations must return a newly allocated map on each call; the
caller may retain and use it directly.
- */
- Map<ComponentVersion, A> load();
+ @Override
+ protected ComponentVersion extractVersion(Class<?> clazz) {
+ UpgradeActionDatanode annotation =
clazz.getAnnotation(UpgradeActionDatanode.class);
+ return annotation.feature();
+ }
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/upgrade/package-info.java
similarity index 83%
copy from
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
copy to
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/upgrade/package-info.java
index 81e1ade3700..17069960958 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/upgrade/package-info.java
@@ -15,12 +15,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.hdds.upgrade;
-
-import org.apache.hadoop.ozone.upgrade.UpgradeAction;
-
/**
- * Upgrade Action for SCM and DataNodes.
+ * Provides Datanode upgrade actions and providers.
*/
-public interface HDDSUpgradeAction<T> extends UpgradeAction<T> {
-}
+package org.apache.hadoop.hdds.upgrade;
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index 5263b7b87a6..977df48a552 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -43,6 +43,7 @@
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
import
org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
+import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeActionProvider;
import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.hdds.utils.IOUtils;
@@ -168,7 +169,7 @@ public DatanodeStateMachine(HddsDatanodeService
hddsDatanodeService,
datanodeDetails.getUuidString());
layoutVersionManager = new HDDSLayoutVersionManager(
- layoutStorage.getApparentVersion());
+ layoutStorage.getApparentVersion(), null, new
DatanodeUpgradeActionProvider());
upgradeFinalizer = new DataNodeUpgradeFinalizer(layoutVersionManager);
VersionedDatanodeFeatures.initialize(layoutVersionManager);
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ContainerTableSchemaFinalizeAction.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ContainerTableSchemaFinalizeAction.java
index 7739797953c..281c68e6641 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ContainerTableSchemaFinalizeAction.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ContainerTableSchemaFinalizeAction.java
@@ -18,10 +18,9 @@
package org.apache.hadoop.ozone.container.upgrade;
import static
org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.WITNESSED_CONTAINER_DB_PROTO_VALUE;
-import static
org.apache.hadoop.ozone.upgrade.UpgradeActionHdds.Component.DATANODE;
import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.upgrade.HDDSUpgradeAction;
+import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeAction;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.CodecException;
import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
@@ -32,16 +31,16 @@
import
org.apache.hadoop.ozone.container.metadata.WitnessedContainerDBDefinition;
import
org.apache.hadoop.ozone.container.metadata.WitnessedContainerMetadataStore;
import
org.apache.hadoop.ozone.container.metadata.WitnessedContainerMetadataStoreImpl;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionHdds;
+import org.apache.hadoop.ozone.upgrade.UpgradeActionDatanode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Upgrade Action for DataNode for update the table schema data of
containerIds Table.
*/
-@UpgradeActionHdds(feature = WITNESSED_CONTAINER_DB_PROTO_VALUE, component =
DATANODE)
+@UpgradeActionDatanode(feature = WITNESSED_CONTAINER_DB_PROTO_VALUE)
public class ContainerTableSchemaFinalizeAction
- implements HDDSUpgradeAction<DatanodeStateMachine> {
+ implements DatanodeUpgradeAction {
private static final Logger LOG =
LoggerFactory.getLogger(ContainerTableSchemaFinalizeAction.class);
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeSchemaV2FinalizeAction.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeSchemaV2FinalizeAction.java
index bf3f5285e36..20005ff9c48 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeSchemaV2FinalizeAction.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeSchemaV2FinalizeAction.java
@@ -18,20 +18,19 @@
package org.apache.hadoop.ozone.container.upgrade;
import static
org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.DATANODE_SCHEMA_V2;
-import static
org.apache.hadoop.ozone.upgrade.UpgradeActionHdds.Component.DATANODE;
-import org.apache.hadoop.hdds.upgrade.HDDSUpgradeAction;
+import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeAction;
import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionHdds;
+import org.apache.hadoop.ozone.upgrade.UpgradeActionDatanode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Upgrade Action for DataNode for the very first first Upgrade Version.
*/
-@UpgradeActionHdds(feature = DATANODE_SCHEMA_V2, component = DATANODE)
+@UpgradeActionDatanode(feature = DATANODE_SCHEMA_V2)
public class DatanodeSchemaV2FinalizeAction
- implements HDDSUpgradeAction<DatanodeStateMachine> {
+ implements DatanodeUpgradeAction {
private static final Logger LOG =
LoggerFactory.getLogger(DatanodeSchemaV2FinalizeAction.class);
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeSchemaV3FinalizeAction.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeSchemaV3FinalizeAction.java
index 6c05a201c2a..e4d97ad5ae4 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeSchemaV3FinalizeAction.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeSchemaV3FinalizeAction.java
@@ -18,16 +18,15 @@
package org.apache.hadoop.ozone.container.upgrade;
import static
org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.DATANODE_SCHEMA_V3;
-import static
org.apache.hadoop.ozone.upgrade.UpgradeActionHdds.Component.DATANODE;
-import org.apache.hadoop.hdds.upgrade.HDDSUpgradeAction;
+import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeAction;
import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionHdds;
+import org.apache.hadoop.ozone.upgrade.UpgradeActionDatanode;
import org.apache.ratis.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,9 +34,9 @@
/**
* Upgrade Action for DataNode for SCHEMA V3.
*/
-@UpgradeActionHdds(feature = DATANODE_SCHEMA_V3, component = DATANODE)
+@UpgradeActionDatanode(feature = DATANODE_SCHEMA_V3)
public class DatanodeSchemaV3FinalizeAction
- implements HDDSUpgradeAction<DatanodeStateMachine> {
+ implements DatanodeUpgradeAction {
private static final Logger LOG =
LoggerFactory.getLogger(DatanodeSchemaV3FinalizeAction.class);
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFinalizeUpgradeActionDatanode.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFinalizeUpgradeActionDatanode.java
index 37d3b241f0d..c8c7ca6d1da 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFinalizeUpgradeActionDatanode.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFinalizeUpgradeActionDatanode.java
@@ -18,28 +18,27 @@
package org.apache.hadoop.ozone.container.upgrade;
import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.SCM_HA;
-import static
org.apache.hadoop.ozone.upgrade.UpgradeActionHdds.Component.DATANODE;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Objects;
-import org.apache.hadoop.hdds.upgrade.HDDSUpgradeAction;
+import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeAction;
import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionHdds;
+import org.apache.hadoop.ozone.upgrade.UpgradeActionDatanode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Action to run upgrade flow for SCM HA exactly once.
*/
-@UpgradeActionHdds(feature = SCM_HA, component = DATANODE)
+@UpgradeActionDatanode(feature = SCM_HA)
public class ScmHAFinalizeUpgradeActionDatanode
- implements HDDSUpgradeAction<DatanodeStateMachine> {
+ implements DatanodeUpgradeAction {
private static final Logger LOG =
LoggerFactory.getLogger(ScmHAFinalizeUpgradeActionDatanode.class);
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionHdds.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionDatanode.java
similarity index 80%
copy from
hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionHdds.java
copy to
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionDatanode.java
index 8809b751f24..f6419085f3b 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionHdds.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionDatanode.java
@@ -24,23 +24,10 @@
import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
/**
- * Annotation to specify upgrade action run during HDDS (SCM or Datanode)
finalization.
+ * Annotation to specify a Datanode upgrade action.
*/
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
-public @interface UpgradeActionHdds {
+public @interface UpgradeActionDatanode {
HDDSLayoutFeature feature();
-
- Component component();
-
- /**
- * Simple enum to denote if an action is for the SCM or the DN.
- */
- enum Component {
- SCM,
- DATANODE;
- }
}
-
-
-
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/upgrade/package-info.java
similarity index 79%
copy from
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
copy to
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/upgrade/package-info.java
index 81e1ade3700..52970af920d 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/upgrade/package-info.java
@@ -15,12 +15,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.hdds.upgrade;
-
-import org.apache.hadoop.ozone.upgrade.UpgradeAction;
-
/**
- * Upgrade Action for SCM and DataNodes.
+ * Provides Datanode upgrade annotations.
*/
-public interface HDDSUpgradeAction<T> extends UpgradeAction<T> {
-}
+package org.apache.hadoop.ozone.upgrade;
diff --git a/hadoop-hdds/docs/content/design/upgrade-dev-primer.md
b/hadoop-hdds/docs/content/design/upgrade-dev-primer.md
index 8934e412f8a..f693c8484c8 100644
--- a/hadoop-hdds/docs/content/design/upgrade-dev-primer.md
+++ b/hadoop-hdds/docs/content/design/upgrade-dev-primer.md
@@ -48,7 +48,7 @@ Method level annotation used to "disallow" an API if current
layout version does
## @BelongsToLayoutVersion Annotation
Annotation to mark an OM request class that it belongs to a specific Layout
Version. Until that version is available post finalize, this request will not
be supported. A newer version of an existing OM request can be created (by
inheritance or a fully new class) and marked with a newer layout version. Until
finalizing this layout version, the older request class is used. Post
finalizing, the newer version of the request class is used.
-## Upgrade Action (UpgradeActionOm & UpgradeActionHdds)
+## Upgrade Action (UpgradeActionOm, UpgradeActionScm & UpgradeActionDatanode)
Annotation to specify upgrade action run during finalization. Each layout
feature can optionally define a single upgrade action that will be executed
when the feature is finalized. This action should be idempotent and execute
quickly. The action must complete for the feature to finish
finalizing, so if there is an error executing the action it will be retried.
This partial failure should not leave the component inoperable.
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java
similarity index 93%
rename from
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java
rename to
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java
index 4e753c5cab3..8cb957aeac1 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java
@@ -27,6 +27,7 @@
import org.apache.hadoop.hdds.ComponentVersion;
import org.apache.hadoop.hdds.HDDSVersion;
import org.apache.hadoop.ozone.upgrade.LayoutFeature;
+import org.apache.hadoop.ozone.upgrade.UpgradeAction;
/**
* List of HDDS Layout Features. All version management has been migrated to
{@link HDDSVersion} and no new additions
@@ -64,8 +65,8 @@ public enum HDDSLayoutFeature implements LayoutFeature {
private final int layoutVersion;
private final String description;
- private HDDSUpgradeAction scmAction;
- private HDDSUpgradeAction datanodeAction;
+ private UpgradeAction<?> scmAction;
+ private UpgradeAction<?> datanodeAction;
HDDSLayoutFeature(final int layoutVersion, String description) {
this.layoutVersion = layoutVersion;
@@ -77,7 +78,7 @@ public enum HDDSLayoutFeature implements LayoutFeature {
*
* @param action The upgrade action to associate with this feature.
*/
- public void addScmAction(HDDSUpgradeAction action) {
+ public void addScmAction(UpgradeAction<?> action) {
// Required by SpotBugs since this setter exists in an enum.
if (this.scmAction == null) {
this.scmAction = action;
@@ -89,7 +90,7 @@ public void addScmAction(HDDSUpgradeAction action) {
*
* @param action The upgrade action to associate with this feature.
*/
- public void addDatanodeAction(HDDSUpgradeAction action) {
+ public void addDatanodeAction(UpgradeAction<?> action) {
// Required by SpotBugs since this setter exists in an enum.
if (this.datanodeAction == null) {
this.datanodeAction = action;
@@ -135,11 +136,11 @@ public String toString() {
return name() + " (" + serialize() + ")";
}
- public Optional<HDDSUpgradeAction> scmAction() {
+ public Optional<UpgradeAction<?>> scmAction() {
return Optional.ofNullable(scmAction);
}
- public Optional<HDDSUpgradeAction> datanodeAction() {
+ public Optional<UpgradeAction<?>> datanodeAction() {
return Optional.ofNullable(datanodeAction);
}
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutVersionManager.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutVersionManager.java
index cfc7fd34618..b890a7c6bf6 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutVersionManager.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutVersionManager.java
@@ -17,14 +17,10 @@
package org.apache.hadoop.hdds.upgrade;
-import static org.apache.hadoop.ozone.upgrade.UpgradeActionHdds.Component.SCM;
-
import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
-import java.util.Set;
import org.apache.hadoop.ozone.upgrade.AbstractLayoutVersionManager;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionHdds;
-import org.reflections.Reflections;
+import org.apache.hadoop.ozone.upgrade.ComponentUpgradeActionProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,17 +34,12 @@ public class HDDSLayoutVersionManager extends
private static final Logger LOG =
LoggerFactory.getLogger(HDDSLayoutVersionManager.class);
- private static final Object[] HDDS_CLASS_UPGRADE_PACKAGES = new Object[] {
- "org.apache.hadoop.hdds.scm.server",
- "org.apache.hadoop.ozone.container",
- };
-
- private static final Reflections HDDS_UPGRADE_REFLECTIONS =
- new Reflections(HDDS_CLASS_UPGRADE_PACKAGES);
- public HDDSLayoutVersionManager(int layoutVersion) throws IOException {
+ public HDDSLayoutVersionManager(int layoutVersion,
+ ComponentUpgradeActionProvider<?> scmProvider,
+ ComponentUpgradeActionProvider<?> dnProvider) throws IOException {
init(layoutVersion, HDDSLayoutFeature.values());
- registerUpgradeActions(getAnnotatedTypes(HDDS_UPGRADE_REFLECTIONS));
+ registerUpgradeActions(scmProvider, dnProvider);
}
public static int maxLayoutVersion() {
@@ -57,46 +48,28 @@ public static int maxLayoutVersion() {
}
@VisibleForTesting
- void registerUpgradeActions(Object... classNames) {
- registerUpgradeActions(getAnnotatedTypes(new Reflections(classNames)));
- }
-
- private static Set<Class<?>> getAnnotatedTypes(Reflections reflections) {
- return reflections.getTypesAnnotatedWith(UpgradeActionHdds.class);
- }
+ void registerUpgradeActions(ComponentUpgradeActionProvider<?> scmProvider,
+ ComponentUpgradeActionProvider<?> dnProvider) {
+ if (scmProvider != null) {
+ scmProvider.load().forEach((feature, action) -> {
+ HDDSLayoutFeature hddsFeature = (HDDSLayoutFeature) feature;
+ if (hddsFeature.layoutVersion() > getMetadataLayoutVersion()) {
+ hddsFeature.addScmAction(action);
+ } else {
+ LOG.debug("Skipping SCM Upgrade Action {} since it has been
finalized.", action.name());
+ }
+ });
+ }
- /**
- * Scan classpath and register all actions to layout features.
- */
- private void registerUpgradeActions(Set<Class<?>> typesAnnotatedWith) {
- typesAnnotatedWith.forEach(actionClass -> {
- if (HDDSUpgradeAction.class.isAssignableFrom(actionClass)) {
- try {
- HDDSUpgradeAction action =
- (HDDSUpgradeAction) actionClass.newInstance();
- UpgradeActionHdds annotation =
- actionClass.getAnnotation(UpgradeActionHdds.class);
- HDDSLayoutFeature feature = annotation.feature();
- if (feature.layoutVersion() > getMetadataLayoutVersion()) {
- LOG.info("Registering Upgrade Action : {}", action.name());
- if (annotation.component() == SCM) {
- feature.addScmAction(action);
- } else {
- feature.addDatanodeAction(action);
- }
- } else {
- LOG.debug("Skipping Upgrade Action {} since it has been finalized"
+
- ".", action.name());
- }
- } catch (Exception e) {
- LOG.error("Cannot instantiate Upgrade Action class {}",
- actionClass.getSimpleName(), e);
+ if (dnProvider != null) {
+ dnProvider.load().forEach((feature, action) -> {
+ HDDSLayoutFeature hddsFeature = (HDDSLayoutFeature) feature;
+ if (hddsFeature.layoutVersion() > getMetadataLayoutVersion()) {
+ hddsFeature.addDatanodeAction(action);
+ } else {
+ LOG.debug("Skipping Datanode Upgrade Action {} since it has been
finalized.", action.name());
}
- } else {
- LOG.warn("Found upgrade action class not of type " +
- "org.apache.hadoop.hdds.upgrade.HDDSUpgradeAction : {}",
- actionClass.getName());
- }
- });
+ });
+ }
}
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractUpgradeActionProvider.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractUpgradeActionProvider.java
new file mode 100644
index 00000000000..8932422f0f3
--- /dev/null
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractUpgradeActionProvider.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.upgrade;
+
+import java.lang.annotation.Annotation;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hadoop.hdds.ComponentVersion;
+import org.reflections.Reflections;
+import org.reflections.scanners.SubTypesScanner;
+import org.reflections.scanners.TypeAnnotationsScanner;
+import org.reflections.util.ConfigurationBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Common abstract provider for loading {@link UpgradeAction} implementations
via reflection.
+ *
+ * @param <T> the concrete upgrade action type
+ */
+public abstract class AbstractUpgradeActionProvider<T extends UpgradeAction<?>>
+ implements ComponentUpgradeActionProvider<T> {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(AbstractUpgradeActionProvider.class);
+
+ private final Class<? extends Annotation> annotationClass;
+ private final Class<T> actionClass;
+ private final String[] packagesToScan;
+
+ protected AbstractUpgradeActionProvider(Class<? extends Annotation>
annotationClass,
+ Class<T> actionClass,
+ String... packagesToScan) {
+ this.annotationClass = annotationClass;
+ this.actionClass = actionClass;
+ this.packagesToScan = packagesToScan;
+ }
+
+ @Override
+ public Map<ComponentVersion, T> load() {
+ Map<ComponentVersion, T> upgradeActions = new HashMap<>();
+
+ Reflections reflections = new Reflections(new ConfigurationBuilder()
+ .forPackages(packagesToScan)
+ .setScanners(new TypeAnnotationsScanner(), new SubTypesScanner())
+ .setExpandSuperTypes(false)
+ .setParallel(true));
+ Set<Class<?>> typesAnnotatedWith =
reflections.getTypesAnnotatedWith(annotationClass);
+
+ typesAnnotatedWith.forEach(clazz -> {
+ if (actionClass.isAssignableFrom(clazz)) {
+ try {
+ @SuppressWarnings("unchecked")
+ T action = (T) clazz.getDeclaredConstructor().newInstance();
+ ComponentVersion feature = extractVersion(clazz);
+ LOG.info("Registering Upgrade Action : {}", action.name());
+ upgradeActions.put(feature, action);
+ } catch (Exception e) {
+ LOG.error("Cannot instantiate Upgrade Action class {}",
+ clazz.getSimpleName(), e);
+ }
+ } else {
+ LOG.warn("Found upgrade action class not of type {} : {}",
+ actionClass.getName(), clazz.getName());
+ }
+ });
+
+ return upgradeActions;
+ }
+
+ /**
+ * Subclasses must implement this to extract the version from the class's
annotation.
+ * Annotation interfaces cannot extend other interfaces, so there is no
common way to extract the version from all
+ * upgrade action annotations for different components.
+ *
+ * @param clazz class annotated with the action annotation
+ * @return ComponentVersion the layout feature associated with the action
+ */
+ protected abstract ComponentVersion extractVersion(Class<?> clazz);
+}
diff --git
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java
similarity index 100%
rename from
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java
rename to
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutVersionManager.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutVersionManager.java
index 4792e1179de..08b4d96c908 100644
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutVersionManager.java
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutVersionManager.java
@@ -19,7 +19,6 @@
import static
org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.DATANODE_SCHEMA_V2;
import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION;
-import static
org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -30,10 +29,12 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Optional;
-import org.apache.hadoop.hdds.upgrade.test.MockComponent;
-import org.apache.hadoop.hdds.upgrade.test.MockComponent.MockDnUpgradeAction;
-import org.apache.hadoop.hdds.upgrade.test.MockComponent.MockScmUpgradeAction;
+import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.ozone.upgrade.ComponentUpgradeActionProvider;
+import org.apache.hadoop.ozone.upgrade.UpgradeAction;
import org.junit.jupiter.api.Test;
/**
@@ -41,44 +42,80 @@
*/
public class TestHDDSLayoutVersionManager {
- private static final String[] UPGRADE_ACTIONS_TEST_PACKAGES = new String[] {
- "org.apache.hadoop.hdds.upgrade.test"};
+ /**
+ * Mock component for testing upgrade actions.
+ */
+ public static class MockComponent {
+ public void mockMethodScm() {
+ }
+
+ public void mockMethodDn() {
+ }
+ }
+
+ /**
+ * Mock SCM upgrade action for testing.
+ */
+ public static class MockScmUpgradeAction implements
UpgradeAction<MockComponent> {
+ @Override
+ public void execute(MockComponent arg) {
+ arg.mockMethodScm();
+ }
+ }
+
+ /**
+ * Mock Datanode upgrade action for testing.
+ */
+ public static class MockDnUpgradeAction implements
UpgradeAction<MockComponent> {
+ @Override
+ public void execute(MockComponent arg) {
+ arg.mockMethodDn();
+ }
+ }
@Test
+ @SuppressWarnings("unchecked")
public void testUpgradeActionsRegistered() throws Exception {
-
- HDDSLayoutVersionManager lvm =
- new HDDSLayoutVersionManager(maxLayoutVersion());
- lvm.registerUpgradeActions(UPGRADE_ACTIONS_TEST_PACKAGES);
+ ComponentUpgradeActionProvider<UpgradeAction<MockComponent>> scmProvider =
() -> {
+ Map<ComponentVersion, UpgradeAction<MockComponent>> map = new
HashMap<>();
+ map.put(INITIAL_VERSION, new MockScmUpgradeAction());
+ return map;
+ };
+
+ ComponentUpgradeActionProvider<UpgradeAction<MockComponent>> dnProvider =
() -> {
+ Map<ComponentVersion, UpgradeAction<MockComponent>> map = new
HashMap<>();
+ map.put(DATANODE_SCHEMA_V2, new MockDnUpgradeAction());
+ return map;
+ };
//Cluster is finalized, hence should not register.
- Optional<HDDSUpgradeAction> action = INITIAL_VERSION.scmAction();
- assertFalse(action.isPresent());
- action = DATANODE_SCHEMA_V2.datanodeAction();
+ Optional<UpgradeAction<?>> action = INITIAL_VERSION.scmAction();
assertFalse(action.isPresent());
+ Optional<UpgradeAction<?>> dnAction = DATANODE_SCHEMA_V2.datanodeAction();
+ assertFalse(dnAction.isPresent());
// Start from an unfinalized version manager.
- lvm = mock(HDDSLayoutVersionManager.class);
+ HDDSLayoutVersionManager lvm = mock(HDDSLayoutVersionManager.class);
when(lvm.getMetadataLayoutVersion()).thenReturn(-1);
- doCallRealMethod().when(lvm).registerUpgradeActions(any());
- lvm.registerUpgradeActions(UPGRADE_ACTIONS_TEST_PACKAGES);
+ doCallRealMethod().when(lvm).registerUpgradeActions(any(), any());
+ lvm.registerUpgradeActions(scmProvider, dnProvider);
action = INITIAL_VERSION.scmAction();
assertTrue(action.isPresent());
assertEquals(MockScmUpgradeAction.class, action.get().getClass());
assertFalse(INITIAL_VERSION.datanodeAction().isPresent());
MockComponent mockObj = mock(MockComponent.class);
- action.get().execute(mockObj);
+ ((UpgradeAction<MockComponent>) action.get()).execute(mockObj);
verify(mockObj, times(1)).mockMethodScm();
verify(mockObj, times(0)).mockMethodDn();
- action = DATANODE_SCHEMA_V2.datanodeAction();
- assertTrue(action.isPresent());
- assertEquals(MockDnUpgradeAction.class, action.get().getClass());
+ dnAction = DATANODE_SCHEMA_V2.datanodeAction();
+ assertTrue(dnAction.isPresent());
+ assertEquals(MockDnUpgradeAction.class, dnAction.get().getClass());
assertFalse(DATANODE_SCHEMA_V2.scmAction().isPresent());
mockObj = mock(MockComponent.class);
- action.get().execute(mockObj);
+ ((UpgradeAction<MockComponent>) dnAction.get()).execute(mockObj);
verify(mockObj, times(0)).mockMethodScm();
verify(mockObj, times(1)).mockMethodDn();
}
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/test/MockComponent.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/test/MockComponent.java
deleted file mode 100644
index b16136c9da7..00000000000
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/test/MockComponent.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.upgrade.test;
-
-import static
org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.DATANODE_SCHEMA_V2;
-import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION;
-import static
org.apache.hadoop.ozone.upgrade.UpgradeActionHdds.Component.DATANODE;
-import static org.apache.hadoop.ozone.upgrade.UpgradeActionHdds.Component.SCM;
-
-import org.apache.hadoop.hdds.upgrade.HDDSUpgradeAction;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionHdds;
-
-/**
- * Mock classes to test upgrade action registration.
- */
-public class MockComponent {
- public void mockMethodScm() {
- }
-
- public void mockMethodDn() {
- }
-
- /**
- * Mock SCM Upgrade Action.
- */
- @UpgradeActionHdds(feature = INITIAL_VERSION,
- component = SCM)
- public static class MockScmUpgradeAction implements
- HDDSUpgradeAction<MockComponent> {
- @Override
- public void execute(MockComponent arg) {
- arg.mockMethodScm();
- }
- }
-
- /**
- * Mock DN Upgrade Action.
- */
- @UpgradeActionHdds(feature = DATANODE_SCHEMA_V2, component = DATANODE)
- public static class MockDnUpgradeAction implements
- HDDSUpgradeAction<MockComponent> {
- @Override
- public void execute(MockComponent arg) {
- arg.mockMethodDn();
- }
- }
-
-}
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java
index 89a52b7c31d..205d8c3c87d 100644
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java
@@ -18,38 +18,17 @@
package org.apache.hadoop.ozone.upgrade;
import java.io.IOException;
-import java.util.Optional;
-import org.apache.hadoop.hdds.upgrade.HDDSUpgradeAction;
-import org.apache.hadoop.hdds.upgrade.test.MockComponent;
/**
* Class to test upgrade related actions.
*/
public class TestUpgradeFinalizerActions {
- /**
- * Mock upgrade finalizer.
- */
- static class MockUpgradeFinalizer extends
- BasicUpgradeFinalizer<MockComponent, MockLayoutVersionManager> {
-
- MockUpgradeFinalizer(MockLayoutVersionManager versionManager) {
- super(versionManager);
+ static class MockComponent {
+ public void mockMethodScm() {
}
- @Override
- public void postFinalizeUpgrade(MockComponent c) {
- return;
- }
-
- @Override
- public void finalizeLayoutFeature(LayoutFeature lf, MockComponent c) {
- return;
- }
-
- @Override
- public void preFinalizeUpgrade(MockComponent c) {
- return;
+ public void mockMethodDn() {
}
}
@@ -70,7 +49,6 @@ enum MockLayoutFeature implements LayoutFeature {
VERSION_3(3);
private int layoutVersion;
- private UpgradeAction action;
MockLayoutFeature(final int layoutVersion) {
this.layoutVersion = layoutVersion;
@@ -97,25 +75,5 @@ public MockLayoutFeature nextVersion() {
public String toString() {
return name() + " (" + serialize() + ")";
}
-
- public void addAction(UpgradeAction upgradeAction) {
- this.action = upgradeAction;
- }
-
- @Override
- public Optional<? extends UpgradeAction> action() {
- return Optional.ofNullable(action);
- }
- }
-
- /**
- * Mock DN Upgrade Action that fails.
- */
- public static class MockFailingUpgradeAction implements
- HDDSUpgradeAction<MockComponent> {
- @Override
- public void execute(MockComponent arg) throws Exception {
- throw new IllegalStateException("Failed action!!");
- }
}
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index db3b7c8eaf0..6acb59eecf0 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -170,6 +170,7 @@
import org.apache.hadoop.hdds.server.http.RatisDropwizardExports;
import org.apache.hadoop.hdds.tracing.TracingConfig;
import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
+import org.apache.hadoop.hdds.upgrade.ScmUpgradeActionProvider;
import org.apache.hadoop.hdds.utils.HAUtils;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.hdds.utils.HddsVersionInfo;
@@ -703,7 +704,7 @@ private void initializeSystemManagers(OzoneConfiguration
conf,
}
scmLayoutVersionManager = new HDDSLayoutVersionManager(
- scmStorageConfig.getApparentVersion());
+ scmStorageConfig.getApparentVersion(), new ScmUpgradeActionProvider(),
null);
VersionedDatanodeFeatures.initialize(scmLayoutVersionManager);
UpgradeFinalizationExecutor<SCMUpgradeFinalizationContext>
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmOnFinalizeActionForDatanodeSchemaV2.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmOnFinalizeActionForDatanodeSchemaV2.java
index fe921a5147e..bd62e87def7 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmOnFinalizeActionForDatanodeSchemaV2.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmOnFinalizeActionForDatanodeSchemaV2.java
@@ -18,19 +18,18 @@
package org.apache.hadoop.hdds.scm.server.upgrade;
import static
org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.DATANODE_SCHEMA_V2;
-import static org.apache.hadoop.ozone.upgrade.UpgradeActionHdds.Component.SCM;
-import org.apache.hadoop.hdds.upgrade.HDDSUpgradeAction;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionHdds;
+import org.apache.hadoop.hdds.upgrade.ScmUpgradeAction;
+import org.apache.hadoop.ozone.upgrade.UpgradeActionScm;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* SCM Upgrade Action for the very first Upgrade Version.
*/
-@UpgradeActionHdds(feature = DATANODE_SCHEMA_V2, component = SCM)
+@UpgradeActionScm(feature = DATANODE_SCHEMA_V2)
public class ScmOnFinalizeActionForDatanodeSchemaV2 implements
- HDDSUpgradeAction<SCMUpgradeFinalizationContext> {
+ ScmUpgradeAction {
private static final Logger LOG =
LoggerFactory.getLogger(ScmOnFinalizeActionForDatanodeSchemaV2.class);
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/upgrade/ScmUpgradeAction.java
similarity index 74%
copy from
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
copy to
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/upgrade/ScmUpgradeAction.java
index 81e1ade3700..25fa5d3e9db 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/upgrade/ScmUpgradeAction.java
@@ -17,10 +17,12 @@
package org.apache.hadoop.hdds.upgrade;
+import org.apache.hadoop.hdds.scm.server.upgrade.SCMUpgradeFinalizationContext;
import org.apache.hadoop.ozone.upgrade.UpgradeAction;
/**
- * Upgrade Action for SCM and DataNodes.
+ * Storage Container Manager Upgrade Action interface. An upgrade action is an
operation that
+ * needs to be executed during finalization.
*/
-public interface HDDSUpgradeAction<T> extends UpgradeAction<T> {
+public interface ScmUpgradeAction extends
UpgradeAction<SCMUpgradeFinalizationContext> {
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentUpgradeActionProvider.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/upgrade/ScmUpgradeActionProvider.java
similarity index 52%
copy from
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentUpgradeActionProvider.java
copy to
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/upgrade/ScmUpgradeActionProvider.java
index bf2a8543d16..63d8dec0b15 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentUpgradeActionProvider.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/upgrade/ScmUpgradeActionProvider.java
@@ -15,24 +15,26 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.upgrade;
+package org.apache.hadoop.hdds.upgrade;
-import java.util.Map;
import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.ozone.upgrade.AbstractUpgradeActionProvider;
+import org.apache.hadoop.ozone.upgrade.UpgradeActionScm;
/**
- * Supplies upgrade actions keyed by {@link ComponentVersion}. Implementations
typically perform classpath scanning or
- * return a fixed map for tests. The component version manager decides when
each action is invoked.
- *
- * @param <A> concrete upgrade action type (for example OM-specific or
HDDS-specific)
+ * Loads {@link ScmUpgradeAction} implementations annotated with {@link
UpgradeActionScm}.
*/
-@FunctionalInterface
-public interface ComponentUpgradeActionProvider<A> {
+public final class ScmUpgradeActionProvider extends
AbstractUpgradeActionProvider<ScmUpgradeAction> {
+
+ public static final String SCM_UPGRADE_CLASS_PACKAGE =
"org.apache.hadoop.hdds.scm.server";
+
+ public ScmUpgradeActionProvider() {
+ super(UpgradeActionScm.class, ScmUpgradeAction.class,
SCM_UPGRADE_CLASS_PACKAGE);
+ }
- /**
- * Returns all upgrade actions from this provider, keyed by component
version.
- * <p>
- * Implementations must return a newly allocated map on each call; the
caller may retain and use it directly.
- */
- Map<ComponentVersion, A> load();
+ @Override
+ protected ComponentVersion extractVersion(Class<?> clazz) {
+ UpgradeActionScm annotation = clazz.getAnnotation(UpgradeActionScm.class);
+ return annotation.feature();
+ }
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/upgrade/package-info.java
similarity index 83%
copy from
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
copy to
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/upgrade/package-info.java
index 81e1ade3700..4af61f6c47b 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/upgrade/package-info.java
@@ -15,12 +15,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.hdds.upgrade;
-
-import org.apache.hadoop.ozone.upgrade.UpgradeAction;
-
/**
- * Upgrade Action for SCM and DataNodes.
+ * Provides SCM upgrade interfaces and implementations.
*/
-public interface HDDSUpgradeAction<T> extends UpgradeAction<T> {
-}
+package org.apache.hadoop.hdds.upgrade;
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionHdds.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionScm.java
similarity index 80%
rename from
hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionHdds.java
rename to
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionScm.java
index 8809b751f24..4cbbcd4e987 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionHdds.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionScm.java
@@ -24,23 +24,10 @@
import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
/**
- * Annotation to specify upgrade action run during HDDS (SCM or Datanode)
finalization.
+ * Annotation to specify an SCM upgrade action.
*/
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
-public @interface UpgradeActionHdds {
+public @interface UpgradeActionScm {
HDDSLayoutFeature feature();
-
- Component component();
-
- /**
- * Simple enum to denote if an action is for the SCM or the DN.
- */
- enum Component {
- SCM,
- DATANODE;
- }
}
-
-
-
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/upgrade/package-info.java
similarity index 79%
rename from
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
rename to
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/upgrade/package-info.java
index 81e1ade3700..e39b1ff28c6 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/upgrade/package-info.java
@@ -15,12 +15,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.hdds.upgrade;
-
-import org.apache.hadoop.ozone.upgrade.UpgradeAction;
-
/**
- * Upgrade Action for SCM and DataNodes.
+ * Provides SCM upgrade actions.
*/
-public interface HDDSUpgradeAction<T> extends UpgradeAction<T> {
-}
+package org.apache.hadoop.ozone.upgrade;
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
index 7250db7db9e..04b6751b8c4 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
@@ -893,7 +893,7 @@ public void testProcessLayoutVersionReportHigherMlv()
when(scmStorageConfig.getClusterID()).thenReturn("xyz111");
EventPublisher eventPublisher = mock(EventPublisher.class);
HDDSLayoutVersionManager lvm =
- new HDDSLayoutVersionManager(scmStorageConfig.getApparentVersion());
+ new HDDSLayoutVersionManager(scmStorageConfig.getApparentVersion(),
null, null);
SCMContext nodeManagerContext = SCMContext.emptyContext();
SCMNodeManager nodeManager = new SCMNodeManager(conf,
scmStorageConfig, eventPublisher, new NetworkTopologyImpl(conf),
@@ -928,7 +928,7 @@ public void testProcessLayoutVersionLowerMlv(boolean
mvlLessThanSlv) throws IOEx
if (mvlLessThanSlv) {
currentVersion -= 1;
}
- HDDSLayoutVersionManager lvm = new
HDDSLayoutVersionManager(currentVersion);
+ HDDSLayoutVersionManager lvm = new
HDDSLayoutVersionManager(currentVersion, null, null);
SCMContext nodeManagerContext = SCMContext.emptyContext();
SCMNodeManager nodeManager = new SCMNodeManager(conf,
@@ -972,7 +972,7 @@ public void testProcessCommandQueueReport()
when(scmStorageConfig.getClusterID()).thenReturn("xyz111");
EventPublisher eventPublisher = mock(EventPublisher.class);
HDDSLayoutVersionManager lvm =
- new HDDSLayoutVersionManager(scmStorageConfig.getApparentVersion());
+ new HDDSLayoutVersionManager(scmStorageConfig.getApparentVersion(),
null, null);
createNodeManager(getConf());
SCMNodeManager nodeManager = new SCMNodeManager(conf,
scmStorageConfig, eventPublisher, new NetworkTopologyImpl(conf),
@@ -2164,7 +2164,7 @@ public void testNodeOperationalStateChange(
SCMStorageConfig scmStorageConfig = mock(SCMStorageConfig.class);
when(scmStorageConfig.getClusterID()).thenReturn("xyz111");
EventPublisher eventPublisher = mock(EventPublisher.class);
- HDDSLayoutVersionManager lvm = new
HDDSLayoutVersionManager(scmStorageConfig.getApparentVersion());
+ HDDSLayoutVersionManager lvm = new
HDDSLayoutVersionManager(scmStorageConfig.getApparentVersion(), null, null);
createNodeManager(getConf());
SCMNodeManager nodeManager = new SCMNodeManager(conf,
scmStorageConfig, eventPublisher, new NetworkTopologyImpl(conf),
diff --git
a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java
b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java
index b609aa0a00e..9ec6852b3e4 100644
---
a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java
+++
b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java
@@ -98,7 +98,7 @@ public static Pair<HDDSLayoutFeature, HDDSLayoutFeature>
getLayoutFeature(
DatanodeLayoutStorage layoutStorage =
new DatanodeLayoutStorage(conf, dnDetail.getUuidString());
HDDSLayoutVersionManager layoutVersionManager =
- new HDDSLayoutVersionManager(layoutStorage.getApparentVersion());
+ new HDDSLayoutVersionManager(layoutStorage.getApparentVersion(), null,
null);
final int metadataLayoutVersion =
layoutVersionManager.getMetadataLayoutVersion();
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeActionProvider.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeActionProvider.java
index 26516a17236..68b0362d1f5 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeActionProvider.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeActionProvider.java
@@ -17,25 +17,14 @@
package org.apache.hadoop.ozone.om.upgrade;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
import org.apache.hadoop.hdds.ComponentVersion;
-import org.apache.hadoop.ozone.upgrade.ComponentUpgradeActionProvider;
-import org.reflections.Reflections;
-import org.reflections.scanners.SubTypesScanner;
-import org.reflections.scanners.TypeAnnotationsScanner;
-import org.reflections.util.ConfigurationBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.hadoop.ozone.upgrade.AbstractUpgradeActionProvider;
/**
* Loads {@link OmUpgradeAction} implementations annotated with {@link
UpgradeActionOm} from
* {@link #OM_UPGRADE_CLASS_PACKAGE} only.
*/
-public final class OMUpgradeActionProvider implements
ComponentUpgradeActionProvider<OmUpgradeAction> {
-
- private static final Logger LOG =
LoggerFactory.getLogger(OMUpgradeActionProvider.class);
+public final class OMUpgradeActionProvider extends
AbstractUpgradeActionProvider<OmUpgradeAction> {
/**
* Package scanned for {@link UpgradeActionOm}-annotated classes (production
OM upgrade actions).
@@ -43,38 +32,12 @@ public final class OMUpgradeActionProvider implements
ComponentUpgradeActionProv
public static final String OM_UPGRADE_CLASS_PACKAGE =
"org.apache.hadoop.ozone.om.upgrade";
public OMUpgradeActionProvider() {
+ super(UpgradeActionOm.class, OmUpgradeAction.class,
OM_UPGRADE_CLASS_PACKAGE);
}
@Override
- public Map<ComponentVersion, OmUpgradeAction> load() {
- Map<ComponentVersion, OmUpgradeAction> upgradeActions = new HashMap<>();
-
- Reflections reflections = new Reflections(new ConfigurationBuilder()
- .forPackages(OM_UPGRADE_CLASS_PACKAGE)
- .setScanners(new TypeAnnotationsScanner(), new SubTypesScanner())
- .setExpandSuperTypes(false)
- .setParallel(true));
- Set<Class<?>> typesAnnotatedWith =
reflections.getTypesAnnotatedWith(UpgradeActionOm.class);
-
- typesAnnotatedWith.forEach(actionClass -> {
- if (OmUpgradeAction.class.isAssignableFrom(actionClass)) {
- try {
- OmUpgradeAction action = (OmUpgradeAction)
actionClass.getDeclaredConstructor().newInstance();
- UpgradeActionOm annotation =
actionClass.getAnnotation(UpgradeActionOm.class);
- OMLayoutFeature feature = annotation.feature();
- LOG.info("Registering Upgrade Action : {}", action.name());
- upgradeActions.put(feature, action);
- } catch (Exception e) {
- LOG.error("Cannot instantiate Upgrade Action class {}",
- actionClass.getSimpleName(), e);
- }
- } else {
- LOG.warn("Found upgrade action class not of type " +
- "org.apache.hadoop.ozone.om.upgrade.OmUpgradeAction : {}",
- actionClass.getName());
- }
- });
-
- return upgradeActions;
+ protected ComponentVersion extractVersion(Class<?> clazz) {
+ UpgradeActionOm annotation = clazz.getAnnotation(UpgradeActionOm.class);
+ return annotation.feature();
}
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
index 5d519ef0410..f44d4e57a1a 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
@@ -216,7 +216,7 @@ public
ReconStorageContainerManagerFacade(OzoneConfiguration conf,
this.dbStore = DBStoreBuilder.createDBStore(ozoneConfiguration,
ReconSCMDBDefinition.get());
HDDSLayoutVersionManager scmLayoutVersionManager =
- new HDDSLayoutVersionManager(scmStorageConfig.getApparentVersion());
+ new HDDSLayoutVersionManager(scmStorageConfig.getApparentVersion(),
null, null);
this.scmhaManager = SCMHAManagerStub.getInstance(
true, new SCMDBTransactionBufferImpl());
this.sequenceIdGen = new SequenceIdGenerator(
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java
index 8aa49a99174..adb09e4ad90 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java
@@ -82,7 +82,7 @@ public void setUp() throws Exception {
ReconUtils reconUtils = new ReconUtils();
ReconStorageConfig reconStorageConfig = new ReconStorageConfig(conf,
reconUtils);
versionManager = new HDDSLayoutVersionManager(
- reconStorageConfig.getApparentVersion());
+ reconStorageConfig.getApparentVersion(), null, null);
store = DBStoreBuilder.createDBStore(conf, ReconSCMDBDefinition.get());
reconContext = new ReconContext(conf, reconUtils);
}
diff --git
a/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java
b/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java
index f3efef5a5a0..797177090a1 100644
---
a/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java
+++
b/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java
@@ -445,7 +445,7 @@ private LayoutVersionProto createLayoutInfo() throws
IOException {
UUID.randomUUID().toString());
HDDSLayoutVersionManager layoutVersionManager =
- new HDDSLayoutVersionManager(layoutStorage.getApparentVersion());
+ new HDDSLayoutVersionManager(layoutStorage.getApparentVersion(), null,
null);
return LayoutVersionProto.newBuilder()
.setMetadataLayoutVersion(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
