This is an automated email from the ASF dual-hosted git repository.
errose28 pushed a commit to branch HDDS-14496-zdu
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-14496-zdu by this push:
new eba78b6c4a8 HDDS-15208. OM should learn to finalize from SCM after
finalization command (#10236)
eba78b6c4a8 is described below
commit eba78b6c4a88ca7e4c13b315121e5b710b0a16bc
Author: Stephen O'Donnell <[email protected]>
AuthorDate: Fri Jun 5 23:48:51 2026 +0100
HDDS-15208. OM should learn to finalize from SCM after finalization command
(#10236)
---
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 3 +
.../common/src/main/resources/ozone-default.xml | 6 +
.../ozone/container/common/DatanodeStorage.java | 4 +-
.../hadoop/ozone/audit/AuditLogTestUtils.java | 26 +-
.../schemaupgrade/UpgradeContainerSchema.java | 35 +--
.../datanode/schemaupgrade/UpgradeUtils.java | 26 --
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 4 +
.../apache/hadoop/fs/ozone/TestHSyncUpgrade.java | 33 +--
.../hadoop/ozone/om/TestOMBucketLayoutUpgrade.java | 19 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 2 +-
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 31 ++-
.../om/multitenant/TestMultiTenantVolume.java | 36 +--
.../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 26 +-
.../src/test/resources/auditlog.properties | 30 ++-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 16 ++
.../request/upgrade/OMFinalizeUpgradeRequest.java | 6 +-
.../upgrade/OMFinalizeUpgradeResponse.java | 3 +
.../ozone/om/upgrade/OMUpgradeFinalizeService.java | 145 +++++++++++
.../upgrade/TestOMFinalizeUpgradeRequest.java | 84 +++++++
.../upgrade/TestOMFinalizeUpgradeResponse.java | 80 +++++++
.../ozone/om/response/upgrade/package-info.java | 21 ++
.../om/upgrade/TestOMUpgradeFinalizeService.java | 265 +++++++++++++++++++++
22 files changed, 763 insertions(+), 138 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 5d0b826c17e..1aa5961eb36 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -439,6 +439,9 @@ public final class OzoneConsts {
// Apparent Version written into Meta Table ONLY during finalization.
// The name "layout version" is kept for backwards compatibility.
public static final String APPARENT_VERSION_KEY = "#LAYOUTVERSION";
+ // Key written into the Meta table when finalization is needed and a
finalization command has been received
+ // to trigger the process
+ public static final String FINALIZATION_IN_PROGRESS_KEY =
"#FINALIZATION_IN_PROGRESS";
// Kerberos constants
public static final String KERBEROS_CONFIG_VALUE = "kerberos";
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index aeda046dcd7..dc16b29b77a 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -5028,4 +5028,10 @@
<tag>OZONE, RATIS, OM</tag>
<description>The maximum number of events that can be pending in OM
Ratis.</description>
</property>
+ <property>
+ <name>ozone.om.upgrade.finalization.check.interval</name>
+ <value>1m</value>
+ <tag>OM</tag>
+ <description>If OM is unfinalized, how frequently it should poll SCM to
trigger finalization</description>
+ </property>
</configuration>
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java
index 53b62afc3e1..fe267783437 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java
@@ -17,12 +17,12 @@
package org.apache.hadoop.ozone.container.common;
-import static
org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion;
import static org.apache.hadoop.ozone.OzoneConsts.DATANODE_LAYOUT_VERSION_DIR;
import java.io.File;
import java.io.IOException;
import java.util.Properties;
+import org.apache.hadoop.hdds.HDDSVersion;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
@@ -95,7 +95,7 @@ public void setClusterId(String clusterId) throws IOException
{
* layout version is found on disk.
*/
private static int getDefaultApparentVersion(ConfigurationSource conf) {
- int defaultApparentVersion = maxLayoutVersion();
+ int defaultApparentVersion = HDDSVersion.SOFTWARE_VERSION.serialize();
File dnIdFile = new File(HddsServerUtil.getDatanodeIdFilePath(conf));
if (dnIdFile.exists()) {
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java
index 0b05d4d01f0..3c438f07c8a 100644
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java
@@ -32,6 +32,7 @@
*/
public final class AuditLogTestUtils {
private static final String AUDITLOG_FILENAME = "audit.log";
+ private static final String SYSTEM_AUDITLOG_FILENAME = "system_audit.log";
private AuditLogTestUtils() {
}
@@ -55,8 +56,26 @@ public static void verifyAuditLog(AuditAction action,
1000, 10000);
}
+ /**
+ * Searches for the given action in the system audit log file.
+ */
+ public static void verifySystemAuditLog(AuditAction action,
+ AuditEventStatus eventStatus) throws InterruptedException,
TimeoutException {
+ waitFor(
+ () -> fileContains(SYSTEM_AUDITLOG_FILENAME, action.getAction(),
eventStatus.getStatus()),
+ 1000, 10000);
+ }
+
public static boolean auditLogContains(String... strings) {
- File file = new File(AUDITLOG_FILENAME);
+ return fileContains(AUDITLOG_FILENAME, strings);
+ }
+
+ public static boolean systemAuditLogContains(String... strings) {
+ return fileContains(SYSTEM_AUDITLOG_FILENAME, strings);
+ }
+
+ private static boolean fileContains(String filename, String... strings) {
+ File file = new File(filename);
try {
String contents = FileUtils.readFileToString(file, UTF_8);
for (String s : strings) {
@@ -72,9 +91,14 @@ public static boolean auditLogContains(String... strings) {
public static void truncateAuditLogFile() throws IOException {
Files.write(Paths.get(AUDITLOG_FILENAME), new byte[0]);
+ Files.write(Paths.get(SYSTEM_AUDITLOG_FILENAME), new byte[0]);
}
public static void deleteAuditLogFile() {
FileUtils.deleteQuietly(new File(AUDITLOG_FILENAME));
}
+
+ public static void deleteSystemAuditLogFile() {
+ FileUtils.deleteQuietly(new File(SYSTEM_AUDITLOG_FILENAME));
+ }
}
diff --git
a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeContainerSchema.java
b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeContainerSchema.java
index 7add67421fa..ffd9dfb8fa4 100644
---
a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeContainerSchema.java
+++
b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeContainerSchema.java
@@ -34,7 +34,7 @@
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.HDDSVersion;
import org.apache.hadoop.hdds.StringUtils;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
@@ -50,6 +50,7 @@
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.Storage;
+import org.apache.hadoop.ozone.container.common.DatanodeStorage;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
@@ -63,6 +64,7 @@
import
org.apache.hadoop.ozone.container.metadata.DatanodeSchemaThreeDBDefinition;
import org.apache.hadoop.ozone.container.metadata.DatanodeStore;
import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl;
+import org.apache.hadoop.ozone.container.upgrade.DatanodeVersionManager;
import org.apache.hadoop.ozone.repair.RepairTool;
import org.apache.hadoop.util.Time;
import picocli.CommandLine;
@@ -131,22 +133,21 @@ public void execute() throws Exception {
DatanodeDetails dnDetail =
UpgradeUtils.getDatanodeDetails(configuration);
- Pair<HDDSLayoutFeature, HDDSLayoutFeature> layoutFeature =
- UpgradeUtils.getLayoutFeature(dnDetail, configuration);
- final HDDSLayoutFeature softwareLayoutFeature = layoutFeature.getLeft();
- final HDDSLayoutFeature metadataLayoutFeature = layoutFeature.getRight();
- final int needLayoutVersion =
- HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion();
-
- if (metadataLayoutFeature.layoutVersion() < needLayoutVersion ||
- softwareLayoutFeature.layoutVersion() < needLayoutVersion) {
- fatal(
- "Please upgrade your software version, no less than %s," +
- " current metadata layout version is %s," +
- " software layout version is %s",
- HDDSLayoutFeature.DATANODE_SCHEMA_V3.toString(),
- metadataLayoutFeature.toString(), softwareLayoutFeature.toString());
- return;
+ DatanodeStorage storage = new DatanodeStorage(configuration,
dnDetail.getUuidString());
+ try (DatanodeVersionManager versionManager = new
DatanodeVersionManager(storage, null)) {
+ // Ensure repair tool is not run in a newer version that supports schema
V3 while the datanode that will read the
+ // containers does not.
+ if
(!HDDSLayoutFeature.DATANODE_SCHEMA_V3.isSupportedBy(HDDSVersion.SOFTWARE_VERSION))
{
+ fatal("Please upgrade your software version to at least %s, current
software version is %s",
+ HDDSLayoutFeature.DATANODE_SCHEMA_V3,
HDDSVersion.SOFTWARE_VERSION);
+ return;
+ }
+
+
+ if (!versionManager.isAllowed(HDDSLayoutFeature.DATANODE_SCHEMA_V3)) {
+ fatal("Please finalize the cluster to enable support for Datanode
container schema V3");
+ return;
+ }
}
if (!Strings.isNullOrEmpty(volume)) {
diff --git
a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java
b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java
index 164f91b6a32..5c559c52f28 100644
---
a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java
+++
b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java
@@ -29,13 +29,9 @@
import java.util.List;
import java.util.Objects;
import java.util.Set;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
-import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
-import org.apache.hadoop.ozone.container.common.DatanodeStorage;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
@@ -93,28 +89,6 @@ public static boolean createFile(File file) throws
IOException {
return file.exists();
}
- public static Pair<HDDSLayoutFeature, HDDSLayoutFeature> getLayoutFeature(
- DatanodeDetails dnDetail, OzoneConfiguration conf) throws IOException {
- DatanodeStorage layoutStorage =
- new DatanodeStorage(conf, dnDetail.getUuidString());
- HDDSLayoutVersionManager layoutVersionManager =
- new HDDSLayoutVersionManager(layoutStorage.getApparentVersion(), null,
null);
-
- final int metadataLayoutVersion =
- layoutVersionManager.getMetadataLayoutVersion();
- final HDDSLayoutFeature metadataLayoutFeature =
- (HDDSLayoutFeature) layoutVersionManager.getFeature(
- metadataLayoutVersion);
-
- final int softwareLayoutVersion =
- layoutVersionManager.getSoftwareLayoutVersion();
- final HDDSLayoutFeature softwareLayoutFeature =
- (HDDSLayoutFeature) layoutVersionManager.getFeature(
- softwareLayoutVersion);
-
- return Pair.of(softwareLayoutFeature, metadataLayoutFeature);
- }
-
public static List<HddsVolume> getAllVolume(DatanodeDetails detail,
OzoneConfiguration configuration) throws IOException {
final MutableVolumeSet dataVolumeSet = getHddsVolumes(configuration,
StorageVolume.VolumeType.DATA_VOLUME,
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index b9ccfedb40d..a9ec011ac07 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -698,6 +698,10 @@ public final class OMConfigKeys {
"ozone.om.ratis.events.max.limit";
public static final int OZONE_OM_RATIS_EVENTS_MAX_LIMIT_DEFAULT = 100;
+ public static final String OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL =
+ "ozone.om.upgrade.finalization.check.interval";
+ public static final String
OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL_DEFAULT = "1m";
+
/**
* Never constructed.
*/
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java
index 22d8f60920f..ff5680a9fd3 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java
@@ -29,16 +29,11 @@
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION;
-import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isDone;
-import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isStarting;
-import static org.apache.ozone.test.LambdaTestUtils.await;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
-import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -53,19 +48,21 @@
import org.apache.hadoop.ozone.ClientConfigForTesting;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.TestDataUtil;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
import org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl;
import org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMStorage;
+import org.apache.hadoop.ozone.om.OMUpgradeTestUtils;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.om.service.OpenKeyCleanupService;
import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
-import org.apache.hadoop.ozone.upgrade.UpgradeFinalization;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -93,9 +90,6 @@ public class TestHSyncUpgrade {
private static final int SERVICE_INTERVAL = 100;
private static final int EXPIRE_THRESHOLD_MS = 140;
- private static final int POLL_INTERVAL_MILLIS = 500;
- private static final int POLL_MAX_WAIT_MILLIS = 120_000;
-
@BeforeEach
public void init() throws Exception {
final BucketLayout layout = BUCKET_LAYOUT;
@@ -118,6 +112,7 @@ public void init() throws Exception {
EXPIRE_THRESHOLD_MS, TimeUnit.MILLISECONDS);
conf.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s");
conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY,
OMLayoutFeature.MULTITENANCY_SCHEMA.layoutVersion());
+ conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL,
"10ms");
ClientConfigForTesting.newBuilder(StorageUnit.BYTES)
.setBlockSize(BLOCK_SIZE)
@@ -217,20 +212,12 @@ private void finalizeOMUpgrade() throws Exception {
// Trigger OM upgrade finalization. Ref: FinalizeUpgradeSubCommand#call
final OzoneManagerProtocol omClient = client.getObjectStore()
.getClientProxy().getOzoneManagerClient();
- final String upgradeClientID = "Test-Upgrade-Client-" + UUID.randomUUID();
- UpgradeFinalization.StatusAndMessages finalizationResponse =
- omClient.finalizeUpgrade(upgradeClientID);
-
- // The status should transition as soon as the client call above returns
- assertTrue(isStarting(finalizationResponse.status()));
- // Wait for the finalization to be marked as done.
- // 10s timeout should be plenty.
- await(POLL_MAX_WAIT_MILLIS, POLL_INTERVAL_MILLIS, () -> {
- final UpgradeFinalization.StatusAndMessages progress =
- omClient.queryUpgradeFinalizationProgress(
- upgradeClientID, false, false);
- return isDone(progress.status());
- });
+ // TODO - OZONE_FINAL_COMMAND - change to sending command when it is
ready. This will trigger OM finalization
+ cluster.getOzoneManager().getMetadataManager().getMetaTable()
+ .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1);
+ cluster.getOzoneManager().getMetadataManager().getMetaTable()
+ .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore");
+ OMUpgradeTestUtils.waitForFinalization(omClient);
}
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java
index 3f4120a4842..3ec9a492acc 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java
@@ -33,8 +33,11 @@
import org.apache.hadoop.hdds.ComponentVersion;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.utils.IOUtils;
+import org.apache.hadoop.hdds.utils.db.CodecException;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.OzoneManagerVersion;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -42,7 +45,6 @@
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
-import org.apache.hadoop.ozone.upgrade.UpgradeFinalization;
import org.apache.ozone.test.LambdaTestUtils;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
@@ -89,6 +91,7 @@ class TestOMBucketLayoutUpgrade {
void setup() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY,
fromVersion.serialize());
+ conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL,
"10ms");
String omServiceId = UUID.randomUUID().toString();
MiniOzoneHAClusterImpl.Builder builder =
MiniOzoneCluster.newHABuilder(conf);
builder.setOMServiceId(omServiceId)
@@ -152,10 +155,16 @@ void allowsLegacyBucketBeforeUpgrade() throws Exception {
@Test
@Order(DURING_UPGRADE)
void finalizeUpgrade() throws Exception {
- UpgradeFinalization.StatusAndMessages response =
- omClient.finalizeUpgrade("finalize-test");
- System.out.println("Finalization Messages : " + response.msgs());
-
+ // TODO - OZONE_FINAL_COMMAND - change to sending command when it is
ready. This will trigger OM finalization
+ cluster.getOzoneManagersList().forEach(om -> {
+ try {
+
om.getMetadataManager().getMetaTable().addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY,
"ignore", 1);
+ om.getMetadataManager().getMetaTable()
+ .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore");
+ } catch (RocksDatabaseException | CodecException e) {
+ throw new RuntimeException(e);
+ }
+ });
waitForFinalization(omClient);
final String expectedVersion =
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
index de2bc98f10c..cdb40ed36d9 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
@@ -297,7 +297,7 @@ public void testInstallSnapshot(@TempDir Path tempDir)
throws Exception {
String toMatch = String.format(
"op=DB_CHECKPOINT_INSTALL
{\"leaderId\":\"%s\",\"term\":\"%d\",\"lastAppliedIndex\":\"%d\"}",
leaderOMNodeId, leaderOMSnapshotTermIndex, followerOMLastAppliedIndex);
- assertTrue(AuditLogTestUtils.auditLogContains(toMatch));
+ assertTrue(AuditLogTestUtils.systemAuditLogContains(toMatch));
// Read & Write after snapshot installed.
List<String> newKeys = writeKeys(1);
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java
index 476ae16af05..03f1786415c 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java
@@ -25,14 +25,18 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import java.io.IOException;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeoutException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.db.CodecException;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.OzoneManagerVersion;
import org.apache.hadoop.ozone.audit.AuditEventStatus;
import org.apache.hadoop.ozone.audit.AuditLogTestUtils;
@@ -40,10 +44,8 @@
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.om.ratis.OzoneManagerStateMachine;
-import org.apache.hadoop.ozone.upgrade.UpgradeFinalization.StatusAndMessages;
import org.apache.ratis.util.LifeCycle;
import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -64,11 +66,14 @@ public void setup() throws Exception {
@AfterAll
public static void shutdown() {
AuditLogTestUtils.deleteAuditLogFile();
+ AuditLogTestUtils.deleteSystemAuditLogFile();
}
@Test
void testOMUpgradeFinalizationWithOneOMDown() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY,
INITIAL_VERSION.layoutVersion());
+ conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL,
"10ms");
try (MiniOzoneHAClusterImpl cluster = newCluster(conf)) {
cluster.waitForClusterToBeReady();
@@ -77,7 +82,7 @@ void testOMUpgradeFinalizationWithOneOMDown() throws
Exception {
for (OzoneManager om : runningOms) {
assertEquals(INITIAL_VERSION,
om.getVersionManager().getApparentVersion());
// The OMs have not been finalized yet, so no version has been
written to the DB.
-
Assertions.assertNull(om.getMetadataManager().getMetaTable().get(APPARENT_VERSION_KEY));
+
assertNull(om.getMetadataManager().getMetaTable().get(APPARENT_VERSION_KEY));
}
final int shutdownOMIndex = 2;
@@ -93,15 +98,25 @@ void testOMUpgradeFinalizationWithOneOMDown() throws
Exception {
long prepareIndex = omClient.prepareOzoneManager(120L, 5L);
assertClusterPrepared(prepareIndex, runningOms);
AuditLogTestUtils.verifyAuditLog(OMAction.UPGRADE_PREPARE,
AuditEventStatus.SUCCESS);
-
omClient.cancelOzoneManagerPrepare();
AuditLogTestUtils.verifyAuditLog(OMAction.UPGRADE_CANCEL,
AuditEventStatus.SUCCESS);
- StatusAndMessages response =
- omClient.finalizeUpgrade("finalize-test");
- System.out.println("Finalization Messages : " + response.msgs());
- AuditLogTestUtils.verifyAuditLog(OMAction.UPGRADE_FINALIZE,
AuditEventStatus.SUCCESS);
+ // TODO - OZONE_FINAL_COMMAND - change to sending command when it is
ready. This will trigger OM finalization
+ cluster.getOzoneManagersList().forEach(om -> {
+ try {
+ om.getMetadataManager().getMetaTable()
+ .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY,
"ignore", 1);
+ om.getMetadataManager().getMetaTable()
+ .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore");
+ } catch (RocksDatabaseException | CodecException e) {
+ throw new RuntimeException(e);
+ }
+ });
waitForFinalization(omClient);
+ AuditLogTestUtils.verifySystemAuditLog(OMAction.UPGRADE_FINALIZE,
AuditEventStatus.SUCCESS);
+ // Ensure the finalization in progress key has been removed.
+ assertNull(cluster.getOzoneManager().getMetadataManager()
+ .getMetaTable().get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY));
cluster.restartOzoneManager(downedOM, true);
OzoneManagerStateMachine omStateMachine = downedOM.getOmRatisServer()
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java
index f1102777327..4fdd713a77d 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java
@@ -18,13 +18,9 @@
package org.apache.hadoop.ozone.om.multitenant;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_MULTITENANCY_ENABLED;
-import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isDone;
-import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isStarting;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
import com.google.protobuf.ServiceException;
import java.io.IOException;
@@ -41,15 +37,15 @@
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.rpc.RpcClient;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMMultiTenantManagerImpl;
import org.apache.hadoop.ozone.om.OMStorage;
+import org.apache.hadoop.ozone.om.OMUpgradeTestUtils;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.om.protocol.S3Auth;
import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
-import org.apache.hadoop.ozone.upgrade.UpgradeFinalization;
-import org.apache.ozone.test.GenericTestUtils;
import org.apache.ozone.test.LambdaTestUtils.VoidCallable;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
@@ -77,6 +73,7 @@ public static void initClusterProvider() throws Exception {
OMMultiTenantManagerImpl.OZONE_OM_TENANT_DEV_SKIP_RANGER, true);
conf.setBoolean(OZONE_OM_MULTITENANCY_ENABLED, true);
conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY,
OMLayoutFeature.INITIAL_VERSION.layoutVersion());
+ conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL,
"10ms");
MiniOzoneCluster.Builder builder = MiniOzoneCluster.newBuilder(conf)
.withoutDatanodes();
cluster = builder.build();
@@ -146,27 +143,12 @@ private static void finalizeOMUpgrade()
// Trigger OM upgrade finalization. Ref: FinalizeUpgradeSubCommand#call
final OzoneManagerProtocol omClient = client.getObjectStore()
.getClientProxy().getOzoneManagerClient();
- final String upgradeClientID = "Test-Upgrade-Client-" + UUID.randomUUID();
- UpgradeFinalization.StatusAndMessages finalizationResponse =
- omClient.finalizeUpgrade(upgradeClientID);
-
- // The status should transition as soon as the client call above returns
- assertTrue(isStarting(finalizationResponse.status()));
-
- // Wait for the finalization to be marked as done.
- // 10s timeout should be plenty.
- GenericTestUtils.waitFor(() -> {
- try {
- final UpgradeFinalization.StatusAndMessages progress =
- omClient.queryUpgradeFinalizationProgress(
- upgradeClientID, false, false);
- return isDone(progress.status());
- } catch (IOException e) {
- fail("Unexpected exception while waiting for "
- + "the OM upgrade to finalize: " + e.getMessage());
- }
- return false;
- }, 500, 10000);
+ // TODO - OZONE_FINAL_COMMAND - change to sending command when it is
ready. This will trigger OM finalization
+ cluster.getOzoneManager().getMetadataManager().getMetaTable()
+ .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1);
+ cluster.getOzoneManager().getMetadataManager().getMetaTable()
+ .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore");
+ OMUpgradeTestUtils.waitForFinalization(omClient);
}
@Test
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
index becf7fb9e5a..7a97d2c28c2 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
@@ -50,8 +50,6 @@
import static
org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.CANCELLED;
import static
org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE;
import static
org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.IN_PROGRESS;
-import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isDone;
-import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isStarting;
import static
org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.COLUMN_FAMILIES_TO_TRACK_IN_DAG;
import static org.apache.ozone.test.LambdaTestUtils.await;
import static org.assertj.core.api.Assertions.assertThat;
@@ -134,6 +132,7 @@
import org.apache.hadoop.ozone.om.KeyManagerImpl;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMStorage;
+import org.apache.hadoop.ozone.om.OMUpgradeTestUtils;
import org.apache.hadoop.ozone.om.OmSnapshot;
import org.apache.hadoop.ozone.om.OmSnapshotManager;
import org.apache.hadoop.ozone.om.OzoneManager;
@@ -153,7 +152,6 @@
import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
-import org.apache.hadoop.ozone.upgrade.UpgradeFinalization;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.ozone.compaction.log.CompactionLogEntry;
@@ -234,6 +232,7 @@ private void init() throws Exception {
// Enable filesystem snapshot feature for the test regardless of the
default
conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true);
conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY,
OMLayoutFeature.BUCKET_LAYOUT_SUPPORT.layoutVersion());
+ conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL,
"10ms");
conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, 1,
TimeUnit.SECONDS);
conf.setInt(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, -1);
conf.setTimeDuration(OZONE_OM_SNAPSHOT_CACHE_CLEANUP_SERVICE_RUN_INTERVAL,
100, TimeUnit.MILLISECONDS);
@@ -324,23 +323,14 @@ private static void
assertFinalizationException(OMException omException) {
* (status FINALIZATION_DONE).
*/
private void finalizeOMUpgrade() throws Exception {
- // Trigger OM upgrade finalization. Ref: FinalizeUpgradeSubCommand#call
final OzoneManagerProtocol omClient = client.getObjectStore()
.getClientProxy().getOzoneManagerClient();
- final String upgradeClientID = "Test-Upgrade-Client-" + UUID.randomUUID();
- UpgradeFinalization.StatusAndMessages finalizationResponse =
- omClient.finalizeUpgrade(upgradeClientID);
-
- // The status should transition as soon as the client call above returns
- assertTrue(isStarting(finalizationResponse.status()));
- // Wait for the finalization to be marked as done.
- // 10s timeout should be plenty.
- await(POLL_MAX_WAIT_MILLIS, POLL_INTERVAL_MILLIS, () -> {
- final UpgradeFinalization.StatusAndMessages progress =
- omClient.queryUpgradeFinalizationProgress(
- upgradeClientID, false, false);
- return isDone(progress.status());
- });
+ // TODO - OZONE_FINAL_COMMAND - change to sending command when it is
ready. This will trigger OM finalization
+ cluster.getOzoneManager().getMetadataManager().getMetaTable()
+ .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1);
+ cluster.getOzoneManager().getMetadataManager().getMetaTable()
+ .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore");
+ OMUpgradeTestUtils.waitForFinalization(omClient);
}
@AfterAll
diff --git
a/hadoop-ozone/integration-test/src/test/resources/auditlog.properties
b/hadoop-ozone/integration-test/src/test/resources/auditlog.properties
index c5e9f8d2c7d..d7051b6cfda 100644
--- a/hadoop-ozone/integration-test/src/test/resources/auditlog.properties
+++ b/hadoop-ozone/integration-test/src/test/resources/auditlog.properties
@@ -52,7 +52,7 @@ filter.write.onMismatch = NEUTRAL
# TRACE (least specific, a lot of data)
# ALL (least specific, all data)
-appenders = console, audit
+appenders = console, audit, systemaudit
appender.console.type = Console
appender.console.name = STDOUT
appender.console.layout.type = PatternLayout
@@ -64,15 +64,27 @@ appender.audit.fileName=audit.log
appender.audit.layout.type=PatternLayout
appender.audit.layout.pattern= %d{DEFAULT} | %-5level | %c{1} | %msg |
%throwable{3} %n
-loggers=audit,omSystemAudit
+loggers=audit,systemaudit
+
logger.audit.name=OMAudit
-logger.audit.level = INFO
-logger.audit.appenderRefs = audit
-logger.audit.appenderRef.file.ref = AUDITLOG
-logger.omSystemAudit.name=OMSystemAudit
-logger.omSystemAudit.level = INFO
-logger.omSystemAudit.appenderRefs = audit
-logger.omSystemAudit.appenderRef.file.ref = AUDITLOG
+logger.audit.level=INFO
+logger.audit.additivity=false
+logger.audit.appenderRefs=audit,console
+logger.audit.appenderRef.audit.ref=AUDITLOG
+logger.audit.appenderRef.console.ref=STDOUT
+
+appender.systemaudit.type = File
+appender.systemaudit.name = SYSTEMAUDITLOG
+appender.systemaudit.fileName=system_audit.log
+appender.systemaudit.layout.type=PatternLayout
+appender.systemaudit.layout.pattern= %d{DEFAULT} | %-5level | %c{1} | %msg |
%throwable{3} %n
+
+logger.systemaudit.name=OMSystemAudit
+logger.systemaudit.level=INFO
+logger.systemaudit.additivity=false
+logger.systemaudit.appenderRefs=systemaudit,console
+logger.systemaudit.appenderRef.systemaudit.ref=SYSTEMAUDITLOG
+logger.systemaudit.appenderRef.console.ref=STDOUT
rootLogger.level = INFO
rootLogger.appenderRefs = stdout
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index f89dae3b3c0..b17eed0bbe3 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -300,6 +300,7 @@
import org.apache.hadoop.ozone.om.service.QuotaRepairTask;
import org.apache.hadoop.ozone.om.snapshot.defrag.SnapshotDefragService;
import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
+import org.apache.hadoop.ozone.om.upgrade.OMUpgradeFinalizeService;
import org.apache.hadoop.ozone.om.upgrade.OMVersionManager;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OzoneManagerAdminService;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
@@ -463,6 +464,7 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
private KeyProviderCryptoExtension kmsProvider;
private final OMVersionManager versionManager;
+ private OMUpgradeFinalizeService omUpgradeFinalizeService;
private final ReplicationConfigValidator replicationConfigValidator;
@@ -639,6 +641,12 @@ private OzoneManager(OzoneConfiguration conf,
StartupOption startupOption)
this.ozoneLockProvider = new OzoneLockProvider(getKeyPathLockEnabled(),
getEnableFileSystemPaths());
+ if (versionManager.needsFinalization()) {
+ long intervalMs =
conf.getTimeDuration(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL,
+ OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS);
+ omUpgradeFinalizeService = new OMUpgradeFinalizeService(this,
versionManager, getScmClient(), intervalMs);
+ }
+
// For testing purpose only, not hit scm from om as Hadoop UGI can't login
// two principals in the same JVM.
ScmInfo scmInfo;
@@ -1900,6 +1908,10 @@ public void start() throws IOException {
bootstrap(omNodeDetails);
}
+ if (omUpgradeFinalizeService != null) {
+ omUpgradeFinalizeService.start();
+ }
+
omState = State.RUNNING;
auditMap.put("NewOmState", omState.name());
SYSTEMAUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMSystemAction.STARTUP,
auditMap));
@@ -2441,6 +2453,10 @@ public boolean stop() {
bucketUtilizationMetrics.unRegister();
}
+ if (omUpgradeFinalizeService != null) {
+ omUpgradeFinalizeService.shutdown();
+ }
+
if (versionManager != null) {
versionManager.close();
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java
index 2bff57e9a8d..0aab6e4cdc3 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java
@@ -25,6 +25,7 @@
import
org.apache.hadoop.hdds.protocol.proto.HddsProtos.UpgradeFinalizationStatus;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.audit.AuditLogger;
import org.apache.hadoop.ozone.audit.OMAction;
import org.apache.hadoop.ozone.om.OMMetadataManager;
@@ -59,7 +60,7 @@ public OMFinalizeUpgradeRequest(OMRequest omRequest) {
@Override
public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
ExecutionContext context) {
LOG.trace("Request: {}", getOmRequest());
- AuditLogger auditLogger = ozoneManager.getAuditLogger();
+ AuditLogger auditLogger = ozoneManager.getSystemAuditLogger();
OzoneManagerProtocolProtos.UserInfo userInfo =
getOmRequest().getUserInfo();
OMResponse.Builder responseBuilder =
OmResponseUtil.getOMResponseBuilder(getOmRequest());
@@ -95,6 +96,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager
ozoneManager, Execut
omMetadataManager.getMetaTable().addCacheEntry(
new CacheKey<>(APPARENT_VERSION_KEY),
CacheValue.get(context.getIndex(), String.valueOf(apparentVersion)));
+ // Clear the finalization_in_progress key from the cache
+ omMetadataManager.getMetaTable().addCacheEntry(
+ new CacheKey<>(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY),
CacheValue.get(context.getIndex()));
FinalizeUpgradeResponse omResponse =
FinalizeUpgradeResponse.newBuilder()
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java
index 1d25f9096f4..8dc981ba5ed 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java
@@ -22,6 +22,7 @@
import java.io.IOException;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
@@ -55,5 +56,7 @@ protected void addToDBBatch(OMMetadataManager
omMetadataManager,
APPARENT_VERSION_KEY,
String.valueOf(serializedApparentVersion));
}
+ // Finalization has completed successfully, so the IN_PROGRESS Key should
be removed from the database.
+ omMetadataManager.getMetaTable().deleteWithBatch(batchOperation,
OzoneConsts.FINALIZATION_IN_PROGRESS_KEY);
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java
new file mode 100644
index 00000000000..93926ba6594
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.upgrade;
+
+import static org.apache.hadoop.ozone.OzoneConsts.FINALIZATION_IN_PROGRESS_KEY;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.ScmClient;
+import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.ratis.protocol.ClientId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A background service that periodically checks whether SCM has completed
finalization of an upgrade and, if so,
+ * finalizes the OM upgrade.
+ */
+public class OMUpgradeFinalizeService extends BackgroundService {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(OMUpgradeFinalizeService.class);
+
+ private static final int THREAD_POOL_SIZE = 1;
+ private static final TimeUnit INTERVAL_UNIT = TimeUnit.MILLISECONDS;
+ private static final long TIMEOUT = 60000;
+ private static final AtomicLong RUN_COUNT = new AtomicLong(0);
+
+ private final OzoneManager ozoneManager;
+ private final OMVersionManager versionManager;
+ private final ScmClient scmClient;
+ private final AtomicBoolean stopInitiated = new AtomicBoolean(false);
+ private final ClientId clientId = ClientId.randomId();
+
+ /**
+ * Creates an {@code OMUpgradeFinalizeService} with a custom check interval.
+ *
+ * @param ozoneManager the OzoneManager instance
+ * @param versionManager the {@link OMVersionManager} to query the
finalization status
+ * @param scmClient the scmClient instance used to query SCM
+ * @param intervalMs the duration to wait between checks
+ */
+ public OMUpgradeFinalizeService(OzoneManager ozoneManager, OMVersionManager
versionManager, ScmClient scmClient,
+ long intervalMs) {
+ super("OMUpgradeFinalizeService", intervalMs, INTERVAL_UNIT,
THREAD_POOL_SIZE, TIMEOUT,
+ ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.versionManager = versionManager;
+ this.scmClient = scmClient;
+ }
+
+ @Override
+ public BackgroundTaskQueue getTasks() {
+ BackgroundTaskQueue queue = new BackgroundTaskQueue();
+ if (!versionManager.needsFinalization()) {
+ // Finalization is done (or was never needed), so this service can now
shutdown. To avoid deadlocking on the
+ // executor.awaitTermination by calling shutdown directly, spawn a
thread to perform the shutdown which will
+ // block until this task / thread completes in the executor.
+ if (stopInitiated.compareAndSet(false, true)) {
+ LOG.info("OMUpgradeFinalizeService: finalization is no longer needed,
shutting down.");
+ Thread stopper = new Thread(this::shutdown,
"OMUpgradeFinalizeService-stopper");
+ stopper.setDaemon(true);
+ stopper.start();
+ }
+ return queue; // empty — PeriodicalTask.run() will return without
scheduling work
+ }
+ if (ozoneManager.isLeaderReady()) {
+ queue.add(new UpgradeStatusCheckTask());
+ }
+ return queue;
+ }
+
+ /**
+ * Periodic task that checks upgrade finalization status and logs the result.
+ */
+ private class UpgradeStatusCheckTask implements BackgroundTask {
+
+ @Override
+ public BackgroundTaskResult call() {
+ final long run = RUN_COUNT.incrementAndGet();
+ if (!ozoneManager.isLeaderReady()) {
+ LOG.debug("OMUpgradeFinalizeService: skipping check — not the leader.
Run count {}", run);
+ return BackgroundTaskResult.EmptyTaskResult.newResult();
+ }
+ if (versionManager.needsFinalization()) {
+ try {
+ // To finalize OM, first finalization needs to have been started.
Then SCM needs to indicate that it has
+ // completed its finalization work. Only once both of those things
have happened can OM finalize.
+ String finalizationInProgress =
+
ozoneManager.getMetadataManager().getMetaTable().get(FINALIZATION_IN_PROGRESS_KEY);
+ if (finalizationInProgress == null) {
+ LOG.debug("OMUpgradeFinalizeService: skipping check — finalization
is not in progress. Run count {}", run);
+ return BackgroundTaskResult.EmptyTaskResult.newResult();
+ }
+
+ HddsProtos.UpgradeStatus upgradeStatus =
scmClient.getContainerClient().queryUpgradeStatus();
+ if (upgradeStatus.getShouldFinalize()) {
+ LOG.info("The SCM Upgrade has been finalized. OM will now
finalize. Run count {}", run);
+
+ OzoneManagerProtocolProtos.OMRequest omRequest =
OzoneManagerProtocolProtos.OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.FinalizeUpgrade)
+ .setClientId(clientId.toString())
+ .build();
+ OzoneManagerProtocolProtos.OMResponse response =
OzoneManagerRatisUtils.submitRequest(
+ ozoneManager, omRequest, clientId, run);
+ if (!response.getSuccess()) {
+ LOG.error("Failed to send FinalizeUpgradeRequest to over Ratis.
{}. Run count {}",
+ response.getMessage(), run);
+ }
+ } else {
+ LOG.debug("The SCM Upgrade has not been finalized. Run count {}",
run);
+ }
+ } catch (Exception e) {
+ LOG.error("An exception occurred while trying to check the SCM
Upgrade status or finalize OM. Run count {}",
+ run, e);
+ }
+ } else {
+ LOG.debug("Finalization is not in progress. Run count {}", run);
+ }
+ return BackgroundTaskResult.EmptyTaskResult.newResult();
+ }
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMFinalizeUpgradeRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMFinalizeUpgradeRequest.java
new file mode 100644
index 00000000000..863ead019ed
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMFinalizeUpgradeRequest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.upgrade;
+
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.Collections;
+import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
+import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.OzoneManagerVersion;
+import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext;
+import org.apache.hadoop.ozone.om.request.key.TestOMKeyRequest;
+import org.apache.hadoop.ozone.om.upgrade.OMVersionManager;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.upgrade.UpgradeFinalization;
+import org.apache.ratis.protocol.ClientId;
+import org.apache.ratis.server.protocol.TermIndex;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for the OMFinalizeUpgradeRequest class.
+ */
+public class TestOMFinalizeUpgradeRequest extends TestOMKeyRequest {
+
+ @Test
+ public void testFinalizationInProgressKeyRemoved() throws IOException {
+ OMVersionManager omVersionManager = mock(OMVersionManager.class);
+
when(omVersionManager.getApparentVersion()).thenReturn(OzoneManagerVersion.DEFAULT_VERSION);
+ when(ozoneManager.getVersionManager()).thenReturn(omVersionManager);
+ when(ozoneManager.finalizeUpgrade(any())).thenReturn(new
UpgradeFinalization.StatusAndMessages(
+ UpgradeFinalization.Status.FINALIZATION_IN_PROGRESS,
Collections.singletonList("Finalization in progress")));
+
+
omMetadataManager.getMetaTable().put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY,
"ignored");
+ omMetadataManager.getMetaTable().addCacheEntry(
+ new CacheKey<>(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY),
CacheValue.get(1, "ignored"));
+
+ String progressKey =
omMetadataManager.getMetaTable().get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY);
+
+ assertNotNull(progressKey);
+ submitRequest();
+
+ progressKey =
omMetadataManager.getMetaTable().get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY);
+ assertNull(progressKey);
+ }
+
+ private void submitRequest() throws IOException {
+ OzoneManagerProtocolProtos.OMRequest omRequest =
OzoneManagerProtocolProtos.OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.FinalizeUpgrade)
+ .setClientId(ClientId.randomId().toString())
+ .build();
+
+ OMFinalizeUpgradeRequest request = new OMFinalizeUpgradeRequest(omRequest);
+ ExecutionContext context = ExecutionContext.of(1, TermIndex.INITIAL_VALUE);
+
+ OzoneManagerProtocolProtos.OMRequest modifiedOmRequest =
request.preExecute(ozoneManager);
+
+ // Will not be equal, as UserInfo will be set.
+ assertNotEquals(omRequest, modifiedOmRequest);
+ request.validateAndUpdateCache(ozoneManager, context);
+ }
+
+}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/TestOMFinalizeUpgradeResponse.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/TestOMFinalizeUpgradeResponse.java
new file mode 100644
index 00000000000..15ad3c848b5
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/TestOMFinalizeUpgradeResponse.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.response.upgrade;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Tests for the OMFinalizeUpgradeResponse class.
+ */
+public class TestOMFinalizeUpgradeResponse {
+
+ @TempDir
+ private Path folder;
+ private OMMetadataManager omMetadataManager;
+ private BatchOperation batchOperation;
+
+ @BeforeEach
+ public void setup() throws Exception {
+ OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+ ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
+ folder.toAbsolutePath().toString());
+ omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null);
+ batchOperation = omMetadataManager.getStore().initBatchOperation();
+ }
+
+ @Test
+ public void testFinalizationInProgressKeyRemoved() throws IOException {
+ // Add the in progress key which would normally have been added by the
start finalization command
+
omMetadataManager.getMetaTable().put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY,
"ignored");
+
+ String value =
omMetadataManager.getMetaTable().get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY);
+ assertEquals("ignored", value);
+
+ OMFinalizeUpgradeResponse finalizeUpgradeResponse = new
OMFinalizeUpgradeResponse(createRequest(), 1);
+ finalizeUpgradeResponse.addToDBBatch(omMetadataManager, batchOperation);
+
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+
+ // Ensure the key is removed as expected
+ value =
omMetadataManager.getMetaTable().get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY);
+ assertNull(value);
+ }
+
+ private OzoneManagerProtocolProtos.OMResponse createRequest() {
+ return OzoneManagerProtocolProtos.OMResponse.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.FinalizeUpgrade)
+ .setStatus(OzoneManagerProtocolProtos.Status.OK)
+ .build();
+ }
+
+}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/package-info.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/package-info.java
new file mode 100644
index 00000000000..ef001e338eb
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package contains test classes for upgrade responses.
+ */
+package org.apache.hadoop.ozone.om.response.upgrade;
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java
new file mode 100644
index 00000000000..a3aa7be8b5f
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.upgrade;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+import com.google.protobuf.ServiceException;
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
+import org.apache.hadoop.hdds.utils.db.CodecException;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.TypedTable;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.OzoneManagerVersion;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.ScmClient;
+import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer;
+import org.apache.ratis.protocol.ClientId;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Unit tests for {@link OMUpgradeFinalizeService}.
+ * Uses {@link
org.apache.hadoop.hdds.utils.BackgroundService#runPeriodicalTaskNow()} to
execute
+ * tasks synchronously on the test thread, avoiding timing-dependent polling.
+ */
+public class TestOMUpgradeFinalizeService {
+
+ // A long interval so the scheduler never fires automatically during tests;
+ // we drive execution manually via runPeriodicalTaskNow().
+ private static final long INTERVAL_MS = 60_000;
+
+ private OzoneManager ozoneManager;
+ private OMVersionManager versionManager;
+ private TypedTable<String, String> metaTable;
+ private ScmClient scmClient;
+ private StorageContainerLocationProtocol containerClient;
+ private OzoneManagerRatisServer omRatisServer;
+ private OMUpgradeFinalizeService service;
+
+ @BeforeEach
+ void setUp() throws RocksDatabaseException, CodecException {
+ ozoneManager = mock(OzoneManager.class);
+ OMMetadataManager metadataManager = mock(OMMetadataManager.class);
+ metaTable = mock(TypedTable.class);
+ when(ozoneManager.getThreadNamePrefix()).thenReturn("");
+ when(ozoneManager.getOMNodeId()).thenReturn("clientId");
+ when(ozoneManager.getMetadataManager()).thenReturn(metadataManager);
+ when(metadataManager.getMetaTable()).thenReturn(metaTable);
+ // For most tests, set the finalization command as having been received
+
when(metaTable.get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY)).thenReturn("ignored");
+
+ versionManager = mock(OMVersionManager.class);
+ // preExecute() calls
ozoneManager.getVersionManager().getApparentVersion().serialize()
+ when(ozoneManager.getVersionManager()).thenReturn(versionManager);
+
when(versionManager.getApparentVersion()).thenReturn(OzoneManagerVersion.DEFAULT_VERSION);
+
+ omRatisServer = mock(OzoneManagerRatisServer.class);
+ // OzoneManagerRatisUtils.submitRequest() calls
ozoneManager.getOmRatisServer().submitRequest(...)
+ when(ozoneManager.getOmRatisServer()).thenReturn(omRatisServer);
+
+ containerClient = mock(StorageContainerLocationProtocol.class);
+ scmClient = mock(ScmClient.class);
+ when(scmClient.getContainerClient()).thenReturn(containerClient);
+
+ service = new OMUpgradeFinalizeService(ozoneManager, versionManager,
scmClient, INTERVAL_MS);
+ }
+
+ /**
+ * When the OM is not the leader, getTasks() should return an empty queue
+ * and no interaction with the SCM client or Ratis server should occur.
+ */
+ @Test
+ void testNoTasksSubmittedWhenNotLeader() throws Exception {
+ when(ozoneManager.isLeaderReady()).thenReturn(false);
+ when(versionManager.needsFinalization()).thenReturn(true);
+
+ service.runPeriodicalTaskNow();
+
+ verifyNoInteractions(scmClient);
+ verifyNoInteractions(omRatisServer);
+ }
+
+ /**
+ * When finalization is not needed, getTasks() should return an empty queue
+ * and no SCM query or Ratis submission should occur.
+ */
+ @Test
+ void testNoTasksSubmittedWhenFinalizationNotNeeded() throws Exception {
+ when(ozoneManager.isLeaderReady()).thenReturn(true);
+ when(versionManager.needsFinalization()).thenReturn(false);
+
+ service.runPeriodicalTaskNow();
+
+ verifyNoInteractions(scmClient);
+ verifyNoInteractions(omRatisServer);
+ }
+
+ /**
+ * When the OM is the leader, finalization is needed, the finalization
command is given and SCM reports
+ * shouldFinalize=true, a FinalizeUpgrade request should be submitted via
Ratis.
+ */
+ @Test
+ void testFinalizationTriggeredWhenScmIsFinalizedAndFinalizationInProgress()
throws Exception {
+ when(ozoneManager.isLeaderReady()).thenReturn(true);
+ when(versionManager.needsFinalization()).thenReturn(true);
+
+ HddsProtos.UpgradeStatus scmStatus = HddsProtos.UpgradeStatus.newBuilder()
+ .setScmFinalized(true)
+ .setShouldFinalize(true)
+ .setNumDatanodesFinalized(3)
+ .setNumDatanodesTotal(3)
+ .build();
+ when(containerClient.queryUpgradeStatus()).thenReturn(scmStatus);
+ // Finalization command not given yet
+
when(metaTable.get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY)).thenReturn(null);
+
+ service.runPeriodicalTaskNow();
+
+ verifyNoInteractions(containerClient);
+ verifyNoInteractions(omRatisServer);
+
+
when(metaTable.get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY)).thenReturn("ignored");
+ service.runPeriodicalTaskNow();
+
+ verify(containerClient).queryUpgradeStatus();
+ // Implementation submits a FinalizeUpgrade request through Ratis
+ verify(omRatisServer).submitRequest(any(), any(ClientId.class), anyLong());
+ }
+
+ /**
+ * When SCM reports shouldFinalize=false (SCM is not yet finalized),
+ * no Ratis request should be submitted.
+ */
+ @Test
+ void testFinalizationSkippedWhenScmNotYetFinalized() throws Exception {
+ when(ozoneManager.isLeaderReady()).thenReturn(true);
+ when(versionManager.needsFinalization()).thenReturn(true);
+
+ HddsProtos.UpgradeStatus scmStatus = HddsProtos.UpgradeStatus.newBuilder()
+ .setScmFinalized(false)
+ .setShouldFinalize(false)
+ .setNumDatanodesFinalized(0)
+ .setNumDatanodesTotal(3)
+ .build();
+ when(containerClient.queryUpgradeStatus()).thenReturn(scmStatus);
+
+ service.runPeriodicalTaskNow();
+
+ verify(containerClient).queryUpgradeStatus();
+ verifyNoInteractions(omRatisServer);
+ }
+
+ /**
+ * When the SCM client throws an IOException, the service should absorb it
+ * and not submit any Ratis request.
+ */
+ @Test
+ void testExceptionFromScmClientIsHandledGracefully() throws Exception {
+ when(ozoneManager.isLeaderReady()).thenReturn(true);
+ when(versionManager.needsFinalization()).thenReturn(true);
+ when(containerClient.queryUpgradeStatus()).thenThrow(new IOException("SCM
unavailable"));
+
+ // The catch block in the task swallows the exception.
+ service.runPeriodicalTaskNow();
+
+ verify(containerClient).queryUpgradeStatus();
+ verifyNoInteractions(omRatisServer);
+ }
+
+ /**
+ * When {@code needsFinalization()} returns {@code false}, {@link
OMUpgradeFinalizeService#getTasks()}
+ * should spawn a stopper thread that calls {@link
OMUpgradeFinalizeService#shutdown()} exactly once,
+ * even when {@code getTasks()} is driven multiple times (guarded by the
internal
+ * {@code stopInitiated} AtomicBoolean). No SCM or Ratis interactions should
occur.
+ * <p>
+ * The service is subclassed to intercept {@code shutdown()} via a {@link
CountDownLatch},
+ * avoiding both actual executor teardown and any need for {@code
Thread.sleep}.
+ */
+ @Test
+ void testShutdownTriggeredExactlyOnceWhenFinalizationNoLongerNeeded() throws
Exception {
+ AtomicInteger shutdownCount = new AtomicInteger(0);
+ CountDownLatch firstShutdown = new CountDownLatch(1);
+
+ OMUpgradeFinalizeService testService = new OMUpgradeFinalizeService(
+ ozoneManager, versionManager, scmClient, INTERVAL_MS) {
+ @Override
+ public synchronized void shutdown() {
+ shutdownCount.incrementAndGet();
+ firstShutdown.countDown();
+ // Don't propagate to super — avoids racing on the test executor.
+ }
+ };
+
+ when(ozoneManager.isLeaderReady()).thenReturn(true);
+ when(versionManager.needsFinalization()).thenReturn(false);
+
+ // Drive getTasks() three times; the stopper thread should only fire once.
+ testService.runPeriodicalTaskNow();
+ testService.runPeriodicalTaskNow();
+ testService.runPeriodicalTaskNow();
+
+ // Wait for the single stopper thread to call shutdown().
+ assertTrue(firstShutdown.await(5, TimeUnit.SECONDS),
+ "shutdown() should have been called by the stopper thread within 5 s");
+ // incrementAndGet() happens before countDown(), so the count is already
final — no sleep needed.
+ assertEquals(1, shutdownCount.get(),
+ "shutdown() must be called exactly once despite multiple getTasks()
invocations");
+ verifyNoInteractions(scmClient);
+ verifyNoInteractions(omRatisServer);
+ }
+
+ /**
+ * When the Ratis submission throws, the service should absorb the error
+ * and not propagate the exception.
+ */
+ @Test
+ void testExceptionFromRatisSubmitIsHandledGracefully() throws Exception {
+ when(ozoneManager.isLeaderReady()).thenReturn(true);
+ when(versionManager.needsFinalization()).thenReturn(true);
+
+ HddsProtos.UpgradeStatus scmStatus = HddsProtos.UpgradeStatus.newBuilder()
+ .setScmFinalized(true)
+ .setShouldFinalize(true)
+ .setNumDatanodesFinalized(3)
+ .setNumDatanodesTotal(3)
+ .build();
+ when(containerClient.queryUpgradeStatus()).thenReturn(scmStatus);
+ when(omRatisServer.submitRequest(any(), any(ClientId.class), anyLong()))
+ .thenThrow(new ServiceException("Ratis unavailable"));
+
+ // The catch block in the task swallows the exception.
+ service.runPeriodicalTaskNow();
+
+ // submitRequest was attempted but threw — service should not propagate
the exception.
+ verify(omRatisServer).submitRequest(any(), any(ClientId.class), anyLong());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
