This is an automated email from the ASF dual-hosted git repository.
devesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e6cadc05c2 HDDS-10813. Improve Recon Debuggability for Snapshot
Fetching, Sync Monitoring, and Permission Validation. (#6640)
e6cadc05c2 is described below
commit e6cadc05c2615750d988ce19b746e4d5f918e648
Author: Arafat2198 <arafatp...@gmail.com>
AuthorDate: Tue Jun 25 13:55:36 2024 +0530
HDDS-10813. Improve Recon Debuggability for Snapshot Fetching, Sync
Monitoring, and Permission Validation. (#6640)
---
.../apache/hadoop/hdds/recon/ReconConfigKeys.java | 3 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 26 +++++
.../spi/impl/OzoneManagerServiceProviderImpl.java | 125 ++++++++++++++++++---
.../ozone/recon/api/TestTriggerDBSyncEndpoint.java | 3 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 101 +++++++++++++++--
5 files changed, 227 insertions(+), 31 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon/ReconConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon/ReconConfigKeys.java
index 3571d39bc8..3ed9f4e58e 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon/ReconConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon/ReconConfigKeys.java
@@ -36,7 +36,8 @@ public final class ReconConfigKeys {
public static final String OZONE_RECON_DB_DIRS_PERMISSIONS =
"ozone.recon.db.dirs.permissions";
-
+ public static final String OZONE_RECON_DB_DIRS_PERMISSIONS_DEFAULT =
+ "750";
public static final String OZONE_RECON_DATANODE_ADDRESS_KEY =
"ozone.recon.datanode.address";
public static final String OZONE_RECON_ADDRESS_KEY =
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java
index fe920b7098..e346b4bc9e 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java
@@ -424,6 +424,32 @@ public class ReconUtils {
}
}
+ /**
+ * Converts Unix numeric permissions into a symbolic representation.
+ * @param numericPermissions The numeric string, e.g., "750".
+ * @return The symbolic representation, e.g., "rwxr-x---".
+ */
+ public static String convertNumericToSymbolic(String numericPermissions) {
+ int owner = Character.getNumericValue(numericPermissions.charAt(0));
+ int group = Character.getNumericValue(numericPermissions.charAt(1));
+ int others = Character.getNumericValue(numericPermissions.charAt(2));
+
+ return String.format("%s%s%s",
+ convertToSymbolicPermission(owner),
+ convertToSymbolicPermission(group),
+ convertToSymbolicPermission(others));
+ }
+
+ /**
+ * Converts a single digit Unix permission into a symbolic representation.
+ * @param permission The permission digit.
+ * @return The symbolic representation for the digit.
+ */
+ public static String convertToSymbolicPermission(int permission) {
+ String[] symbols = {"---", "--x", "-w-", "-wx", "r--", "r-x", "rw-",
"rwx"};
+ return symbols[permission];
+ }
+
/**
* Sorts a list of DiskUsage objects in descending order by size using
parallel sorting and
* returns the top N records as specified by the limit.
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
index 26b9bec9d6..bde89eea1d 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
@@ -23,16 +23,23 @@ import javax.inject.Singleton;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Arrays;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.hdds.recon.ReconConfigKeys;
import org.apache.hadoop.hdds.utils.db.RocksDatabase;
import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteOptions;
@@ -50,6 +57,7 @@ import org.apache.hadoop.ozone.om.helpers.DBUpdates;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort.Type;
+import org.apache.hadoop.ozone.recon.ReconContext;
import org.apache.hadoop.ozone.recon.ReconServerConfigKeys;
import org.apache.hadoop.ozone.recon.ReconUtils;
import org.apache.hadoop.ozone.recon.metrics.OzoneManagerSyncMetrics;
@@ -63,6 +71,8 @@ import org.apache.hadoop.util.Time;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.FileUtils;
+
+import static
org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DB_DIRS_PERMISSIONS_DEFAULT;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE;
@@ -81,6 +91,7 @@ import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.RECON_OM_DELTA
import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.RECON_OM_DELTA_UPDATE_LIMIT_DEFUALT;
import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.RECON_OM_DELTA_UPDATE_LOOP_LIMIT;
import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.RECON_OM_DELTA_UPDATE_LOOP_LIMIT_DEFUALT;
+import static
org.apache.hadoop.ozone.recon.ReconUtils.convertNumericToSymbolic;
import static org.apache.ratis.proto.RaftProtos.RaftPeerRole.LEADER;
import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
@@ -101,6 +112,7 @@ public class OzoneManagerServiceProviderImpl
private URLConnectionFactory connectionFactory;
private File omSnapshotDBParentDir = null;
+ private File reconDbDir = null;
private String omDBSnapshotUrl;
private OzoneManagerProtocol ozoneManagerClient;
@@ -119,6 +131,7 @@ public class OzoneManagerServiceProviderImpl
private AtomicBoolean isSyncDataFromOMRunning;
private final String threadNamePrefix;
private ThreadFactory threadFactory;
+ private ReconContext reconContext;
/**
* OM Snapshot related task names.
@@ -134,7 +147,8 @@ public class OzoneManagerServiceProviderImpl
ReconOMMetadataManager omMetadataManager,
ReconTaskController reconTaskController,
ReconUtils reconUtils,
- OzoneManagerProtocol ozoneManagerClient) {
+ OzoneManagerProtocol ozoneManagerClient,
+ ReconContext reconContext) {
int connectionTimeout = (int) configuration.getTimeDuration(
OZONE_RECON_OM_CONNECTION_TIMEOUT,
@@ -168,6 +182,8 @@ public class OzoneManagerServiceProviderImpl
omSnapshotDBParentDir = reconUtils.getReconDbDir(configuration,
OZONE_RECON_OM_SNAPSHOT_DB_DIR);
+ reconDbDir = reconUtils.getReconDbDir(configuration,
+ ReconConfigKeys.OZONE_RECON_DB_DIR);
HttpConfig.Policy policy = HttpConfig.getHttpPolicy(configuration);
@@ -205,6 +221,7 @@ public class OzoneManagerServiceProviderImpl
this.threadFactory =
new ThreadFactoryBuilder().setNameFormat(threadNamePrefix +
"SyncOM-%d")
.build();
+ this.reconContext = reconContext;
}
public void registerOMDBTasks() {
@@ -242,7 +259,7 @@ public class OzoneManagerServiceProviderImpl
try {
omMetadataManager.start(configuration);
} catch (IOException ioEx) {
- LOG.error("Error staring Recon OM Metadata Manager.", ioEx);
+ LOG.error("Error starting Recon OM Metadata Manager.", ioEx);
}
reconTaskController.start();
long initialDelay = configuration.getTimeDuration(
@@ -264,10 +281,12 @@ public class OzoneManagerServiceProviderImpl
LOG.debug("Started the OM DB sync scheduler.");
scheduler.scheduleWithFixedDelay(() -> {
try {
+ LOG.info("Last known sequence number before sync: {}",
getCurrentOMDBSequenceNumber());
boolean isSuccess = syncDataFromOM();
if (!isSuccess) {
LOG.debug("OM DB sync is already running.");
}
+ LOG.info("Sequence number after sync: {}",
getCurrentOMDBSequenceNumber());
} catch (Throwable t) {
LOG.error("Unexpected exception while syncing data from OM.", t);
}
@@ -361,16 +380,30 @@ public class OzoneManagerServiceProviderImpl
return null;
});
// Untar the checkpoint file.
- Path untarredDbDir = Paths.get(omSnapshotDBParentDir.getAbsolutePath(),
- snapshotFileName);
+ Path untarredDbDir = Paths.get(omSnapshotDBParentDir.getAbsolutePath(),
snapshotFileName);
reconUtils.untarCheckpointFile(targetFile, untarredDbDir);
FileUtils.deleteQuietly(targetFile);
+ // Validate the presence of required SST files
+ File[] sstFiles = untarredDbDir.toFile().listFiles((dir, name) ->
name.endsWith(".sst"));
+ if (sstFiles == null || sstFiles.length == 0) {
+ LOG.warn("No SST files found in the OM snapshot directory: {}",
untarredDbDir);
+ }
+
+ List<String> sstFileNames = Arrays.stream(sstFiles)
+ .map(File::getName)
+ .collect(Collectors.toList());
+ LOG.debug("Valid SST files found: {}", sstFileNames);
+
// Currently, OM DB type is not configurable. Hence, defaulting to
// RocksDB.
+ reconContext.updateHealthStatus(new AtomicBoolean(true));
+
reconContext.getErrors().remove(ReconContext.ErrorCode.GET_OM_DB_SNAPSHOT_FAILED);
return new RocksDBCheckpoint(untarredDbDir);
} catch (IOException e) {
LOG.error("Unable to obtain Ozone Manager DB Snapshot. ", e);
+ reconContext.updateHealthStatus(new AtomicBoolean(false));
+
reconContext.updateErrors(ReconContext.ErrorCode.GET_OM_DB_SNAPSHOT_FAILED);
}
return null;
}
@@ -381,25 +414,36 @@ public class OzoneManagerServiceProviderImpl
*/
@VisibleForTesting
boolean updateReconOmDBWithNewSnapshot() throws IOException {
+ // Check permissions of the Recon DB directory
+ checkAndValidateReconDbPermissions();
// Obtain the current DB snapshot from OM and
// update the in house OM metadata managed DB instance.
long startTime = Time.monotonicNow();
DBCheckpoint dbSnapshot = getOzoneManagerDBSnapshot();
metrics.updateSnapshotRequestLatency(Time.monotonicNow() - startTime);
- if (dbSnapshot != null && dbSnapshot.getCheckpointLocation() != null) {
- LOG.info("Got new checkpoint from OM : " +
- dbSnapshot.getCheckpointLocation());
- try {
- omMetadataManager.updateOmDB(
- dbSnapshot.getCheckpointLocation().toFile());
- return true;
- } catch (IOException e) {
- LOG.error("Unable to refresh Recon OM DB Snapshot. ", e);
- }
- } else {
- LOG.error("Null snapshot location got from OM.");
+
+ if (dbSnapshot == null) {
+ LOG.error("Failed to obtain a valid DB snapshot from Ozone Manager. This
could be due to " +
+ "missing SST files or other fetch issues.");
+ return false;
+ }
+
+ if (dbSnapshot.getCheckpointLocation() == null) {
+ LOG.error("Snapshot checkpoint location is null, indicating a failure to
properly fetch or " +
+ "store the snapshot.");
+ return false;
+ }
+
+ LOG.info("Attempting to update Recon OM DB with new snapshot located at:
{}",
+ dbSnapshot.getCheckpointLocation());
+ try {
+
omMetadataManager.updateOmDB(dbSnapshot.getCheckpointLocation().toFile());
+ LOG.info("Successfully updated Recon OM DB with new snapshot.");
+ return true;
+ } catch (IOException e) {
+ LOG.error("Unable to refresh Recon OM DB Snapshot.", e);
+ return false;
}
- return false;
}
/**
@@ -549,24 +593,69 @@ public class OzoneManagerServiceProviderImpl
// Reinitialize tasks that are listening.
LOG.info("Calling reprocess on Recon tasks.");
reconTaskController.reInitializeTasks(omMetadataManager);
+
+ // Update health status in ReconContext
+ reconContext.updateHealthStatus(new AtomicBoolean(true));
+
reconContext.getErrors().remove(ReconContext.ErrorCode.GET_OM_DB_SNAPSHOT_FAILED);
+ } else {
+ metrics.incrNumSnapshotRequestsFailed();
+ // Update health status in ReconContext
+ reconContext.updateHealthStatus(new AtomicBoolean(false));
+
reconContext.updateErrors(ReconContext.ErrorCode.GET_OM_DB_SNAPSHOT_FAILED);
}
} catch (InterruptedException intEx) {
Thread.currentThread().interrupt();
} catch (Exception e) {
metrics.incrNumSnapshotRequestsFailed();
LOG.error("Unable to update Recon's metadata with new OM DB. ", e);
+ // Update health status in ReconContext
+ reconContext.updateHealthStatus(new AtomicBoolean(false));
+
reconContext.updateErrors(ReconContext.ErrorCode.GET_OM_DB_SNAPSHOT_FAILED);
}
}
} finally {
isSyncDataFromOMRunning.set(false);
}
} else {
- LOG.debug("OM DB sync is already running.");
+ LOG.info("OM DB sync is already running.");
return false;
}
return true;
}
+ public void checkAndValidateReconDbPermissions() {
+ File dbDir = new File(reconDbDir.getPath());
+ if (!dbDir.exists()) {
+ LOG.error("Recon DB directory does not exist: {}",
dbDir.getAbsolutePath());
+ return;
+ }
+
+ try {
+ // Fetch expected minimum permissions from configuration
+ String expectedPermissions =
+ configuration.get(ReconConfigKeys.OZONE_RECON_DB_DIRS_PERMISSIONS,
OZONE_RECON_DB_DIRS_PERMISSIONS_DEFAULT);
+ Set<PosixFilePermission> expectedPosixPermissions =
+
PosixFilePermissions.fromString(convertNumericToSymbolic(expectedPermissions));
+
+ // Get actual permissions
+ Set<PosixFilePermission> actualPermissions =
Files.getPosixFilePermissions(dbDir.toPath());
+ String actualPermissionsStr =
PosixFilePermissions.toString(actualPermissions);
+
+ // Check if actual permissions meet the minimum required permissions
+ if (actualPermissions.containsAll(expectedPosixPermissions)) {
+ LOG.info("Permissions for Recon DB directory '{}' meet the minimum
required permissions '{}'",
+ dbDir.getAbsolutePath(), expectedPermissions);
+ } else {
+ LOG.warn("Permissions for Recon DB directory '{}' are '{}', which do
not meet the minimum" +
+ " required permissions '{}'", dbDir.getAbsolutePath(),
actualPermissionsStr, expectedPermissions);
+ }
+ } catch (IOException e) {
+ LOG.error("Failed to retrieve permissions for Recon DB directory: {}",
dbDir.getAbsolutePath(), e);
+ } catch (IllegalArgumentException e) {
+ LOG.error("Configuration issue: {}", e.getMessage());
+ }
+ }
+
/**
* Get OM RocksDB's latest sequence number.
* @return latest sequence number.
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTriggerDBSyncEndpoint.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTriggerDBSyncEndpoint.java
index 9bf824e4ee..7ad5dcc58c 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTriggerDBSyncEndpoint.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTriggerDBSyncEndpoint.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.ozone.om.helpers.DBUpdates;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.recon.MetricsServiceProviderFactory;
+import org.apache.hadoop.ozone.recon.ReconContext;
import org.apache.hadoop.ozone.recon.ReconTestInjector;
import org.apache.hadoop.ozone.recon.ReconUtils;
import org.apache.hadoop.ozone.recon.common.CommonUtils;
@@ -121,7 +122,7 @@ public class TestTriggerDBSyncEndpoint {
OzoneManagerServiceProviderImpl ozoneManagerServiceProvider =
new OzoneManagerServiceProviderImpl(configuration,
reconOMMetadataManager, reconTaskController, reconUtilsMock,
- ozoneManagerProtocol);
+ ozoneManagerProtocol, new ReconContext(configuration,
reconUtilsMock));
ozoneManagerServiceProvider.start();
reconTestInjector =
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java
index 032bff80ad..2700034aae 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.DBUpdates;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.recon.ReconContext;
import org.apache.hadoop.ozone.recon.ReconUtils;
import org.apache.hadoop.ozone.recon.common.CommonUtils;
import org.apache.hadoop.ozone.recon.metrics.OzoneManagerSyncMetrics;
@@ -91,6 +92,7 @@ public class TestOzoneManagerServiceProviderImpl {
private OzoneConfiguration configuration;
private OzoneManagerProtocol ozoneManagerProtocol;
private CommonUtils commonUtils;
+ private ReconContext reconContext;
@BeforeEach
public void setUp(@TempDir File dirReconSnapDB, @TempDir File dirReconDB)
@@ -103,6 +105,7 @@ public class TestOzoneManagerServiceProviderImpl {
configuration.set("ozone.om.address", "localhost:9862");
ozoneManagerProtocol = getMockOzoneManagerClient(new DBUpdates());
commonUtils = new CommonUtils();
+ reconContext = new ReconContext(configuration, new ReconUtils());
}
@Test
@@ -136,7 +139,7 @@ public class TestOzoneManagerServiceProviderImpl {
OzoneManagerServiceProviderImpl ozoneManagerServiceProvider =
new OzoneManagerServiceProviderImpl(configuration,
reconOMMetadataManager, reconTaskController, reconUtilsMock,
- ozoneManagerProtocol);
+ ozoneManagerProtocol, reconContext);
assertNull(reconOMMetadataManager.getKeyTable(getBucketLayout())
.get("/sampleVol/bucketOne/key_one"));
@@ -145,6 +148,81 @@ public class TestOzoneManagerServiceProviderImpl {
assertTrue(ozoneManagerServiceProvider.updateReconOmDBWithNewSnapshot());
+ assertNotNull(reconOMMetadataManager.getKeyTable(getBucketLayout())
+ .get("/sampleVol/bucketOne/key_one"));
+ assertNotNull(reconOMMetadataManager.getKeyTable(getBucketLayout())
+ .get("/sampleVol/bucketOne/key_two"));
+
+ // Verifying if context error GET_OM_DB_SNAPSHOT_FAILED is removed
+
assertFalse(reconContext.getErrors().contains(ReconContext.ErrorCode.GET_OM_DB_SNAPSHOT_FAILED));
+ }
+
+ @Test
+ public void testUpdateReconOmDBWithNewSnapshotFailure(
+ @TempDir File dirOmMetadata, @TempDir File dirReconMetadata)
+ throws Exception {
+
+ OMMetadataManager omMetadataManager =
+ initializeNewOmMetadataManager(dirOmMetadata);
+ ReconOMMetadataManager reconOMMetadataManager =
+ getTestReconOmMetadataManager(omMetadataManager,
+ dirReconMetadata);
+
+ ReconUtils reconUtilsMock = getMockReconUtils();
+
+ when(reconUtilsMock.makeHttpCall(any(), anyString(), anyBoolean()))
+ .thenThrow(new IOException("Mocked IOException"));
+ when(reconUtilsMock.getReconNodeDetails(
+ any(OzoneConfiguration.class))).thenReturn(
+ commonUtils.getReconNodeDetails());
+ ReconTaskController reconTaskController = getMockTaskController();
+
+ OzoneManagerServiceProviderImpl ozoneManagerServiceProvider =
+ new OzoneManagerServiceProviderImpl(configuration,
+ reconOMMetadataManager, reconTaskController, reconUtilsMock,
+ ozoneManagerProtocol, reconContext);
+
+ assertFalse(ozoneManagerServiceProvider.updateReconOmDBWithNewSnapshot());
+
+ // Verifying if context error GET_OM_DB_SNAPSHOT_FAILED is added
+
assertTrue(reconContext.getErrors().contains(ReconContext.ErrorCode.GET_OM_DB_SNAPSHOT_FAILED));
+ }
+
+ @Test
+ public void testUpdateReconOmDBWithNewSnapshotSuccess(
+ @TempDir File dirOmMetadata, @TempDir File dirReconMetadata) throws
Exception {
+
+ OMMetadataManager omMetadataManager =
+ initializeNewOmMetadataManager(dirOmMetadata);
+ ReconOMMetadataManager reconOMMetadataManager =
+ getTestReconOmMetadataManager(omMetadataManager, dirReconMetadata);
+
+ writeDataToOm(omMetadataManager, "key_one");
+ writeDataToOm(omMetadataManager, "key_two");
+
+ DBCheckpoint checkpoint = omMetadataManager.getStore().getCheckpoint(true);
+ File tarFile = createTarFile(checkpoint.getCheckpointLocation());
+ InputStream inputStream = new FileInputStream(tarFile);
+ ReconUtils reconUtilsMock = getMockReconUtils();
+ HttpURLConnection httpURLConnectionMock = mock(HttpURLConnection.class);
+ when(httpURLConnectionMock.getInputStream()).thenReturn(inputStream);
+ when(reconUtilsMock.makeHttpCall(any(), anyString(), anyBoolean()))
+ .thenReturn(httpURLConnectionMock);
+ when(reconUtilsMock.getReconNodeDetails(any(OzoneConfiguration.class)))
+ .thenReturn(commonUtils.getReconNodeDetails());
+ ReconTaskController reconTaskController = getMockTaskController();
+
+
reconContext.updateErrors(ReconContext.ErrorCode.GET_OM_DB_SNAPSHOT_FAILED);
+
+ OzoneManagerServiceProviderImpl ozoneManagerServiceProvider =
+ new OzoneManagerServiceProviderImpl(configuration,
+ reconOMMetadataManager, reconTaskController, reconUtilsMock,
+ ozoneManagerProtocol, reconContext);
+
+
assertTrue(reconContext.getErrors().contains(ReconContext.ErrorCode.GET_OM_DB_SNAPSHOT_FAILED));
+ assertTrue(ozoneManagerServiceProvider.updateReconOmDBWithNewSnapshot());
+
assertFalse(reconContext.getErrors().contains(ReconContext.ErrorCode.GET_OM_DB_SNAPSHOT_FAILED));
+
assertNotNull(reconOMMetadataManager.getKeyTable(getBucketLayout())
.get("/sampleVol/bucketOne/key_one"));
assertNotNull(reconOMMetadataManager.getKeyTable(getBucketLayout())
@@ -182,7 +260,7 @@ public class TestOzoneManagerServiceProviderImpl {
OzoneManagerServiceProviderImpl ozoneManagerServiceProvider1 =
new OzoneManagerServiceProviderImpl(configuration,
reconOMMetadataManager, reconTaskController, reconUtilsMock,
- ozoneManagerProtocol);
+ ozoneManagerProtocol, reconContext);
assertTrue(ozoneManagerServiceProvider1.updateReconOmDBWithNewSnapshot());
HttpURLConnection httpURLConnectionMock2 = mock(HttpURLConnection.class);
@@ -192,7 +270,7 @@ public class TestOzoneManagerServiceProviderImpl {
OzoneManagerServiceProviderImpl ozoneManagerServiceProvider2 =
new OzoneManagerServiceProviderImpl(configuration,
reconOMMetadataManager, reconTaskController, reconUtilsMock,
- ozoneManagerProtocol);
+ ozoneManagerProtocol, reconContext);
assertTrue(ozoneManagerServiceProvider2.updateReconOmDBWithNewSnapshot());
}
@@ -238,7 +316,7 @@ public class TestOzoneManagerServiceProviderImpl {
OzoneManagerServiceProviderImpl ozoneManagerServiceProvider =
new OzoneManagerServiceProviderImpl(configuration,
reconOMMetadataManager, reconTaskController, reconUtilsMock,
- ozoneManagerProtocol);
+ ozoneManagerProtocol, reconContext);
DBCheckpoint checkpoint = ozoneManagerServiceProvider
.getOzoneManagerDBSnapshot();
@@ -288,7 +366,7 @@ public class TestOzoneManagerServiceProviderImpl {
new OzoneManagerServiceProviderImpl(configuration,
getTestReconOmMetadataManager(omMetadataManager, dirReconMetadata),
getMockTaskController(), new ReconUtils(),
- getMockOzoneManagerClient(dbUpdatesWrapper));
+ getMockOzoneManagerClient(dbUpdatesWrapper), reconContext);
OMDBUpdatesHandler updatesHandler =
new OMDBUpdatesHandler(omMetadataManager);
@@ -358,7 +436,7 @@ public class TestOzoneManagerServiceProviderImpl {
getTestReconOmMetadataManager(omMetadataManager, dirReconMetadata),
getMockTaskController(), new ReconUtils(),
getMockOzoneManagerClientWith4Updates(dbUpdatesWrapper[0],
- dbUpdatesWrapper[1], dbUpdatesWrapper[2],
dbUpdatesWrapper[3]));
+ dbUpdatesWrapper[1], dbUpdatesWrapper[2],
dbUpdatesWrapper[3]), reconContext);
assertTrue(dbUpdatesWrapper[0].isDBUpdateSuccess());
assertTrue(dbUpdatesWrapper[1].isDBUpdateSuccess());
@@ -414,7 +492,7 @@ public class TestOzoneManagerServiceProviderImpl {
OzoneManagerServiceProviderImpl ozoneManagerServiceProvider =
new MockOzoneServiceProvider(configuration, omMetadataManager,
- reconTaskControllerMock, new ReconUtils(), ozoneManagerProtocol);
+ reconTaskControllerMock, new ReconUtils(), ozoneManagerProtocol,
reconContext);
OzoneManagerSyncMetrics metrics = ozoneManagerServiceProvider.getMetrics();
assertEquals(0, metrics.getNumSnapshotRequests());
@@ -454,7 +532,7 @@ public class TestOzoneManagerServiceProviderImpl {
OzoneManagerServiceProviderImpl ozoneManagerServiceProvider =
new OzoneManagerServiceProviderImpl(configuration, omMetadataManager,
- reconTaskControllerMock, new ReconUtils(), ozoneManagerProtocol);
+ reconTaskControllerMock, new ReconUtils(), ozoneManagerProtocol,
reconContext);
OzoneManagerSyncMetrics metrics = ozoneManagerServiceProvider.getMetrics();
@@ -495,7 +573,7 @@ public class TestOzoneManagerServiceProviderImpl {
OzoneManagerProtocol protocol = getMockOzoneManagerClientWithThrow();
OzoneManagerServiceProviderImpl ozoneManagerServiceProvider =
new MockOzoneServiceProvider(configuration, omMetadataManager,
- reconTaskControllerMock, new ReconUtils(), protocol);
+ reconTaskControllerMock, new ReconUtils(), protocol, reconContext);
OzoneManagerSyncMetrics metrics = ozoneManagerServiceProvider.getMetrics();
@@ -569,9 +647,10 @@ class MockOzoneServiceProvider extends
OzoneManagerServiceProviderImpl {
ReconOMMetadataManager omMetadataManager,
ReconTaskController reconTaskController,
ReconUtils reconUtils,
- OzoneManagerProtocol ozoneManagerClient) {
+ OzoneManagerProtocol ozoneManagerClient,
+ ReconContext reconContext) {
super(configuration, omMetadataManager, reconTaskController, reconUtils,
- ozoneManagerClient);
+ ozoneManagerClient, reconContext);
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@ozone.apache.org
For additional commands, e-mail: commits-h...@ozone.apache.org
