This is an automated email from the ASF dual-hosted git repository.
devmadhuu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ca3c9798d2b HDDS-15413. Recon and SCM Container Sync Metrics addition.
(#10384)
ca3c9798d2b is described below
commit ca3c9798d2b6ef2048f1986902502e16f147b6f1
Author: Devesh Kumar Singh <[email protected]>
AuthorDate: Wed Jun 10 15:34:10 2026 +0530
HDDS-15413. Recon and SCM Container Sync Metrics addition. (#10384)
---
.../ozone/recon/TestNSSummaryMemoryLeak.java | 5 +-
.../TestReconContainerHealthSummaryEndToEnd.java | 2 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 2 +-
.../metrics/ReconScmContainerSyncMetrics.java | 182 +++++++++++++++++----
.../scm/ReconStorageContainerManagerFacade.java | 33 ++--
.../recon/scm/ReconStorageContainerSyncHelper.java | 53 +++++-
.../metrics/TestReconScmContainerSyncMetrics.java | 99 +++++++++++
.../scm/TestReconSCMContainerSyncIntegration.java | 19 ++-
.../TestReconStorageContainerManagerFacade.java | 117 +++++++++++++
.../scm/TestReconStorageContainerSyncHelper.java | 50 +++++-
10 files changed, 503 insertions(+), 59 deletions(-)
diff --git
a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryMemoryLeak.java
b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryMemoryLeak.java
index fdb0a3c1204..34ba542b1de 100644
---
a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryMemoryLeak.java
+++
b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryMemoryLeak.java
@@ -240,7 +240,7 @@ public void testNSSummaryCleanupOnHardDelete() throws
Exception {
// This simulates the background process that hard deletes entries
simulateHardDelete(omMetadataManager);
- // Verify cleanup after hard delete simulation.
+ // Verify memory leak fix - NSSummary entries should be cleaned up
verifyNSSummaryCleanup(omMetadataManager, "memoryLeakTest");
LOG.info("NSSummary memory leak fix test completed successfully");
@@ -309,8 +309,9 @@ public void testMemoryLeakWithLargeStructure() throws
Exception {
// Simulate hard delete
simulateHardDelete(omMetadataManager);
+ syncDataFromOM();
- // Verify cleanup after hard delete simulation.
+ // Verify NSSummary cleanup
verifyNSSummaryCleanup(omMetadataManager, "largeMemoryLeakTest");
LOG.info("Large structure memory leak test completed successfully");
diff --git
a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerHealthSummaryEndToEnd.java
b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerHealthSummaryEndToEnd.java
index f1976f5a878..af60f46a60b 100644
---
a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerHealthSummaryEndToEnd.java
+++
b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerHealthSummaryEndToEnd.java
@@ -1182,7 +1182,7 @@ private void syncAndWaitForReconContainers(
ReconStorageContainerManagerFacade reconScm,
ReconContainerManager reconCm,
List<ContainerID> containerIDs) throws Exception {
- reconScm.triggerTargetedSCMContainerSync();
+ reconScm.triggerSCMContainerSync();
drainScmAndReconEventQueues();
backfillMissingContainersFromScm(reconCm, containerIDs);
LambdaTestUtils.await(REPLICA_SYNC_TIMEOUT_MS, POLL_INTERVAL_MS,
diff --git
a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java
b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java
index c3e213c07ea..f847812edcd 100644
---
a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java
+++
b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java
@@ -185,7 +185,7 @@ public void testSyncSCMContainerInfo() throws Exception {
int scmContainersCount = scmContainerManager.getContainers().size();
int reconContainersCount = reconCm.getContainers().size();
assertNotEquals(scmContainersCount, reconContainersCount);
- reconScm.triggerTargetedSCMContainerSync();
+ reconScm.triggerSCMContainerSync();
reconContainersCount = reconCm.getContainers().size();
assertEquals(scmContainersCount, reconContainersCount);
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/ReconScmContainerSyncMetrics.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/ReconScmContainerSyncMetrics.java
index d652a63b4ef..67d4520fd91 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/ReconScmContainerSyncMetrics.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/ReconScmContainerSyncMetrics.java
@@ -17,50 +17,78 @@
package org.apache.hadoop.ozone.recon.metrics;
+import com.google.common.base.CaseFormat;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.MetricsSource;
import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
-import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
+import org.apache.hadoop.metrics2.lib.Interns;
import org.apache.hadoop.ozone.OzoneConsts;
/**
- * Metrics for Recon SCM targeted sync execution.
+ * Metrics for Recon SCM container sync execution.
*/
@InterfaceAudience.Private
@Metrics(about = "Recon SCM Container Sync Metrics", context =
OzoneConsts.OZONE)
-public final class ReconScmContainerSyncMetrics {
+public final class ReconScmContainerSyncMetrics implements MetricsSource {
private static final String SOURCE_NAME =
ReconScmContainerSyncMetrics.class.getSimpleName();
+ private static final HddsProtos.LifeCycleState[] SYNC_STATES = {
+ HddsProtos.LifeCycleState.OPEN,
+ HddsProtos.LifeCycleState.QUASI_CLOSED,
+ HddsProtos.LifeCycleState.CLOSED,
+ HddsProtos.LifeCycleState.DELETED
+ };
+
+ private static final MetricsInfo SCM_CONTAINER_SYNC_STATUS = Interns.info(
+ "scmContainerSyncStatus",
+ "SCM container sync status: 0=idle, 1=in progress, 2=success,
3=failure");
+
+ private static final MetricsInfo SCM_CONTAINER_SYNC_DURATION_MS =
Interns.info(
+ "scmContainerSyncDurationMs",
+ "Time taken by the SCM container sync in milliseconds");
+
/**
- * No targeted sync has run yet, or the latest scheduler cycle did not run
one.
- */
- public static final int TARGETED_SYNC_STATUS_IDLE = 0;
- /**
- * Targeted sync is currently running.
+ * SCM container sync is currently running.
*/
- public static final int TARGETED_SYNC_STATUS_IN_PROGRESS = 1;
+ public static final int SCM_CONTAINER_SYNC_STATUS_IN_PROGRESS = 1;
/**
- * The last targeted sync completed successfully.
+ * SCM container sync completed successfully.
*/
- public static final int TARGETED_SYNC_STATUS_SUCCESS = 2;
+ public static final int SCM_CONTAINER_SYNC_STATUS_SUCCESS = 2;
/**
- * The last targeted sync completed with one or more failed passes.
+ * SCM container sync completed with one or more failed passes.
*/
- public static final int TARGETED_SYNC_STATUS_FAILURE = 3;
-
- @Metric(about = "Targeted sync status: 0=idle, 1=in progress, "
- + "2=success, 3=failure")
- private MutableGaugeInt targetedSyncStatus;
+ public static final int SCM_CONTAINER_SYNC_STATUS_FAILURE = 3;
- @Metric(about = "Time taken by the last targeted sync in milliseconds")
- private MutableGaugeLong lastTargetedSyncDurationMs;
+ private final AtomicInteger scmContainerSyncStatus = new AtomicInteger();
+ private final AtomicLong scmContainerSyncDurationMs = new AtomicLong();
+ private final Map<HddsProtos.LifeCycleState, AtomicLong>
+ containerSyncDurationMs;
+ private final Map<HddsProtos.LifeCycleState, AtomicLong>
+ containerCountDrift;
+ private final Map<HddsProtos.LifeCycleState, MetricsInfo>
+ containerSyncDurationMetricInfo;
+ private final Map<HddsProtos.LifeCycleState, MetricsInfo>
+ containerCountDriftMetricInfo;
private ReconScmContainerSyncMetrics() {
+ containerSyncDurationMs = initStateGaugeValues();
+ containerCountDrift = initStateGaugeValues();
+ containerSyncDurationMetricInfo = initSyncDurationMetricInfo();
+ containerCountDriftMetricInfo = initCountDriftMetricInfo();
}
public static ReconScmContainerSyncMetrics create() {
@@ -75,19 +103,115 @@ public void unRegister() {
ms.unregisterSource(SOURCE_NAME);
}
- public void setTargetedSyncStatus(int status) {
- targetedSyncStatus.set(status);
+ public void setScmContainerSyncStatus(int status) {
+ scmContainerSyncStatus.set(status);
+ }
+
+ public void setScmContainerSyncDurationMs(long durationMs) {
+ scmContainerSyncDurationMs.set(durationMs);
+ }
+
+ public void setContainerSyncDurationMs(
+ HddsProtos.LifeCycleState state, long durationMs) {
+ setStateGauge(containerSyncDurationMs, state, durationMs);
+ }
+
+ public void setContainerCountDrift(
+ HddsProtos.LifeCycleState state, long drift) {
+ setStateGauge(containerCountDrift, state, drift);
+ }
+
+ public int getScmContainerSyncStatus() {
+ return scmContainerSyncStatus.get();
+ }
+
+ public long getScmContainerSyncDurationMs() {
+ return scmContainerSyncDurationMs.get();
+ }
+
+ public long getContainerSyncDurationMs(
+ HddsProtos.LifeCycleState state) {
+ return getStateGauge(containerSyncDurationMs, state);
+ }
+
+ public long getContainerCountDrift(
+ HddsProtos.LifeCycleState state) {
+ return getStateGauge(containerCountDrift, state);
+ }
+
+ @Override
+ public void getMetrics(MetricsCollector collector, boolean all) {
+ MetricsRecordBuilder builder = collector.addRecord(SOURCE_NAME);
+ builder.addGauge(SCM_CONTAINER_SYNC_STATUS, getScmContainerSyncStatus());
+ builder.addGauge(SCM_CONTAINER_SYNC_DURATION_MS,
+ getScmContainerSyncDurationMs());
+ for (HddsProtos.LifeCycleState state : SYNC_STATES) {
+ builder.addGauge(containerSyncDurationMetricInfo.get(state),
+ getContainerSyncDurationMs(state));
+ builder.addGauge(containerCountDriftMetricInfo.get(state),
+ getContainerCountDrift(state));
+ }
+ }
+
+ private static Map<HddsProtos.LifeCycleState, AtomicLong>
+ initStateGaugeValues() {
+ Map<HddsProtos.LifeCycleState, AtomicLong> gauges =
+ new EnumMap<>(HddsProtos.LifeCycleState.class);
+ for (HddsProtos.LifeCycleState state : SYNC_STATES) {
+ gauges.put(state, new AtomicLong());
+ }
+ return Collections.unmodifiableMap(gauges);
+ }
+
+ private static Map<HddsProtos.LifeCycleState, MetricsInfo>
+ initSyncDurationMetricInfo() {
+ Map<HddsProtos.LifeCycleState, MetricsInfo> metrics =
+ new EnumMap<>(HddsProtos.LifeCycleState.class);
+ for (HddsProtos.LifeCycleState state : SYNC_STATES) {
+ String stateName = metricStateName(state);
+ metrics.put(state, Interns.info(
+ CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_CAMEL, stateName)
+ + "ContainerSyncDurationMs",
+ "Time taken by the " + stateName
+ + " container sync pass in milliseconds"));
+ }
+ return Collections.unmodifiableMap(metrics);
+ }
+
+ private static Map<HddsProtos.LifeCycleState, MetricsInfo>
+ initCountDriftMetricInfo() {
+ Map<HddsProtos.LifeCycleState, MetricsInfo> metrics =
+ new EnumMap<>(HddsProtos.LifeCycleState.class);
+ for (HddsProtos.LifeCycleState state : SYNC_STATES) {
+ String stateName = metricStateName(state);
+ metrics.put(state, Interns.info(
+ CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_CAMEL, stateName)
+ + "ContainerCountDrift",
+ "Last successfully observed container count drift at start of sync
pass "
+ + "(SCM count minus Recon count for " + stateName + " state)."));
+ }
+ return Collections.unmodifiableMap(metrics);
}
- public void setLastTargetedSyncDurationMs(long durationMs) {
- lastTargetedSyncDurationMs.set(durationMs);
+ private static String metricStateName(HddsProtos.LifeCycleState state) {
+ return CaseFormat.UPPER_UNDERSCORE.to(
+ CaseFormat.UPPER_CAMEL, state.name());
}
- public int getTargetedSyncStatus() {
- return targetedSyncStatus.value();
+ private static void setStateGauge(
+ Map<HddsProtos.LifeCycleState, AtomicLong> gauges,
+ HddsProtos.LifeCycleState state,
+ long value) {
+ AtomicLong gauge = gauges.get(state);
+ if (gauge != null) {
+ gauge.set(value);
+ }
}
- public long getLastTargetedSyncDurationMs() {
- return lastTargetedSyncDurationMs.value();
+ private static long getStateGauge(
+ Map<HddsProtos.LifeCycleState, AtomicLong> gauges,
+ HddsProtos.LifeCycleState state) {
+ AtomicLong gauge = gauges.get(state);
+ return gauge != null ? gauge.get() : 0L;
}
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
index 679e3c3c71e..73f76e4731a 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
@@ -548,7 +548,8 @@ public
ReconStorageContainerManagerFacade(OzoneConfiguration conf,
containerSyncHelper = new ReconStorageContainerSyncHelper(
scmServiceProvider,
ozoneConfiguration,
- containerManager
+ containerManager,
+ containerSyncMetrics
);
}
@@ -597,8 +598,8 @@ public void start() {
initializePipelinesFromScm();
}
// -----------------------------------------------------------------------
- // Scheduler (incremental/targeted sync): runs on the configured interval.
- // Each cycle directly runs targeted reconciliation. The sync itself
already
+ // Scheduler (SCM container sync): runs on the configured interval.
+ // Each cycle directly runs SCM container reconciliation. The sync itself
already
// fetches the SCM state counts needed for pagination, so a separate drift
// preflight would duplicate SCM calls before doing the same work.
// -----------------------------------------------------------------------
@@ -617,9 +618,9 @@ public void start() {
return;
}
try {
- boolean success = runTargetedSyncWithMetrics();
+ boolean success = runScmContainerSyncWithMetrics();
if (!success) {
- LOG.warn("Targeted sync completed with one or more phase failures. "
+ LOG.warn("SCM container sync completed with one or more phase
failures. "
+ "Check logs above for details.");
}
} catch (Throwable t) {
@@ -883,10 +884,10 @@ private void cleanupFailedOrCancelledCheckpoint(File
checkpointLocation,
* Runs targeted reconciliation immediately rather than waiting for the next
* scheduled cycle.
*/
- public boolean triggerTargetedSCMContainerSync() {
+ public boolean triggerSCMContainerSync() {
if (isSyncDataFromSCMRunning.compareAndSet(false, true)) {
try {
- return runTargetedSyncWithMetrics();
+ return runScmContainerSyncWithMetrics();
} finally {
isSyncDataFromSCMRunning.compareAndSet(true, false);
}
@@ -896,22 +897,22 @@ public boolean triggerTargetedSCMContainerSync() {
}
}
- private boolean runTargetedSyncWithMetrics() {
+ private boolean runScmContainerSyncWithMetrics() {
long startTime = Time.monotonicNow();
- containerSyncMetrics.setTargetedSyncStatus(
- ReconScmContainerSyncMetrics.TARGETED_SYNC_STATUS_IN_PROGRESS);
+ containerSyncMetrics.setScmContainerSyncStatus(
+ ReconScmContainerSyncMetrics.SCM_CONTAINER_SYNC_STATUS_IN_PROGRESS);
try {
boolean success = containerSyncHelper.syncWithSCMContainerInfo();
- containerSyncMetrics.setTargetedSyncStatus(success
- ? ReconScmContainerSyncMetrics.TARGETED_SYNC_STATUS_SUCCESS
- : ReconScmContainerSyncMetrics.TARGETED_SYNC_STATUS_FAILURE);
+ containerSyncMetrics.setScmContainerSyncStatus(success
+ ? ReconScmContainerSyncMetrics.SCM_CONTAINER_SYNC_STATUS_SUCCESS
+ : ReconScmContainerSyncMetrics.SCM_CONTAINER_SYNC_STATUS_FAILURE);
return success;
} catch (RuntimeException | Error e) {
- containerSyncMetrics.setTargetedSyncStatus(
- ReconScmContainerSyncMetrics.TARGETED_SYNC_STATUS_FAILURE);
+ containerSyncMetrics.setScmContainerSyncStatus(
+ ReconScmContainerSyncMetrics.SCM_CONTAINER_SYNC_STATUS_FAILURE);
throw e;
} finally {
- containerSyncMetrics.setLastTargetedSyncDurationMs(
+ containerSyncMetrics.setScmContainerSyncDurationMs(
Time.monotonicNow() - startTime);
}
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerSyncHelper.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerSyncHelper.java
index bbd03aab0d5..1a7d4e28d00 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerSyncHelper.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerSyncHelper.java
@@ -33,6 +33,7 @@
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -42,7 +43,9 @@
import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import
org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException;
+import org.apache.hadoop.ozone.recon.metrics.ReconScmContainerSyncMetrics;
import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
+import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -121,6 +124,8 @@ class ReconStorageContainerSyncHelper {
*/
private static final long CONTAINER_WITH_PIPELINE_PROTO_SIZE_BYTES = 1024;
+ private static final int LIVE_STATE_SYNC_PROGRESS_LOG_INTERVAL = 50;
+
/**
* Monotonic cursor for OPEN add-only sync. OPEN containers are
* created with increasing container IDs, so each cycle only needs to scan
@@ -141,13 +146,17 @@ class ReconStorageContainerSyncHelper {
private final StorageContainerServiceProvider scmServiceProvider;
private final OzoneConfiguration ozoneConfiguration;
private final ReconContainerManager containerManager;
+ private final ReconScmContainerSyncMetrics containerSyncMetrics;
ReconStorageContainerSyncHelper(StorageContainerServiceProvider
scmServiceProvider,
OzoneConfiguration ozoneConfiguration,
- ReconContainerManager containerManager) {
+ ReconContainerManager containerManager,
+ ReconScmContainerSyncMetrics
containerSyncMetrics) {
this.scmServiceProvider = scmServiceProvider;
this.ozoneConfiguration = ozoneConfiguration;
this.containerManager = containerManager;
+ this.containerSyncMetrics =
+ Objects.requireNonNull(containerSyncMetrics, "containerSyncMetrics");
}
/**
@@ -168,8 +177,10 @@ public boolean syncWithSCMContainerInfo() {
*/
private boolean syncContainersForState(HddsProtos.LifeCycleState scmState,
boolean incrementalOpen) {
+ long startTime = Time.monotonicNow();
try {
long total = scmServiceProvider.getContainerCount(scmState);
+ updateContainerCountDrift(scmState, total);
if (total == 0) {
LOG.debug("{} sync: no containers found in SCM.", scmState);
return true;
@@ -181,7 +192,10 @@ private boolean
syncContainersForState(HddsProtos.LifeCycleState scmState,
long retrieved = 0;
int addedCount = 0;
int reconciledCount = 0;
+ int batchCount = 0;
+ LOG.info("{} sync starting: total={}, batchSize={}, startId={}.",
+ scmState, total, batchSize, initialStart);
while (true) {
List<ContainerID> batch = scmServiceProvider.getListOfContainerIDs(
startContainerId, batchSize, scmState);
@@ -215,6 +229,14 @@ private boolean
syncContainersForState(HddsProtos.LifeCycleState scmState,
}
startContainerId = ContainerID.valueOf(nextID);
retrieved += batch.size();
+ batchCount++;
+
+ if (batchCount % LIVE_STATE_SYNC_PROGRESS_LOG_INTERVAL == 0) {
+ LOG.info("{} sync progress: batch={}, totalRetrieved={}, added={}, "
+ + "reconciled={}, nextId={}.",
+ scmState, batchCount, retrieved, addedCount, reconciledCount,
+ nextID);
+ }
}
LOG.info("{} sync complete from start {}, checked {}, added {},
reconciled {}.",
@@ -223,6 +245,8 @@ private boolean
syncContainersForState(HddsProtos.LifeCycleState scmState,
} catch (Exception e) {
LOG.error("{} sync: unexpected error.", scmState, e);
return false;
+ } finally {
+ updateContainerSyncDuration(scmState, Time.monotonicNow() - startTime);
}
}
@@ -360,7 +384,9 @@ private int rebuildContainerFromScm(ContainerID containerID,
* @return {@code true} if all RPC calls completed without error
*/
private boolean syncDeletedContainers() {
+ long startTime = Time.monotonicNow();
try {
+ updateDeletedContainerCountDrift();
int configuredBatch = ozoneConfiguration.getInt(
OZONE_RECON_SCM_DELETED_CONTAINER_CHECK_BATCH_SIZE,
OZONE_RECON_SCM_DELETED_CONTAINER_CHECK_BATCH_SIZE_DEFAULT);
@@ -393,9 +419,34 @@ private boolean syncDeletedContainers() {
} catch (Exception e) {
LOG.error("DELETED sync: unexpected error.", e);
return false;
+ } finally {
+ updateContainerSyncDuration(HddsProtos.LifeCycleState.DELETED,
+ Time.monotonicNow() - startTime);
+ }
+ }
+
+ private void updateDeletedContainerCountDrift() {
+ try {
+ long total = scmServiceProvider.getContainerCount(
+ HddsProtos.LifeCycleState.DELETED);
+ updateContainerCountDrift(HddsProtos.LifeCycleState.DELETED, total);
+ } catch (Exception e) {
+ LOG.warn("DELETED sync: unable to update pre-sync count drift metric.",
e);
}
}
+ private void updateContainerCountDrift(HddsProtos.LifeCycleState state,
+ long scmCount) {
+ long reconCount = containerManager.getContainerStateCount(state);
+ containerSyncMetrics.setContainerCountDrift(state,
+ scmCount - reconCount);
+ }
+
+ private void updateContainerSyncDuration(HddsProtos.LifeCycleState state,
+ long durationMs) {
+ containerSyncMetrics.setContainerSyncDurationMs(state, durationMs);
+ }
+
/**
* Processes one page of DELETED container IDs from SCM.
* For each container:
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/metrics/TestReconScmContainerSyncMetrics.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/metrics/TestReconScmContainerSyncMetrics.java
new file mode 100644
index 00000000000..6208a7cd96a
--- /dev/null
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/metrics/TestReconScmContainerSyncMetrics.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.metrics;
+
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSED;
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSING;
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.DELETED;
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.DELETING;
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.OPEN;
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.QUASI_CLOSED;
+import static org.apache.hadoop.metrics2.lib.Interns.info;
+import static org.apache.ozone.test.MetricsAsserts.eqName;
+import static org.apache.ozone.test.MetricsAsserts.getIntGauge;
+import static org.apache.ozone.test.MetricsAsserts.getLongGauge;
+import static org.apache.ozone.test.MetricsAsserts.getMetrics;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for Recon SCM container sync metrics.
+ */
+class TestReconScmContainerSyncMetrics {
+
+ private ReconScmContainerSyncMetrics metrics;
+
+ @BeforeEach
+ void setUp() {
+ metrics = ReconScmContainerSyncMetrics.create();
+ }
+
+ @AfterEach
+ void tearDown() {
+ metrics.unRegister();
+ }
+
+ @Test
+ void testStateMetricsAreEmittedForReconciledStatesOnly() {
+ metrics.setContainerSyncDurationMs(OPEN, 10L);
+ metrics.setContainerSyncDurationMs(QUASI_CLOSED, 20L);
+ metrics.setContainerSyncDurationMs(CLOSED, 30L);
+ metrics.setContainerSyncDurationMs(DELETED, 40L);
+ metrics.setContainerCountDrift(OPEN, 2L);
+ metrics.setContainerCountDrift(QUASI_CLOSED, 0L);
+ metrics.setContainerCountDrift(CLOSED, -3L);
+ metrics.setContainerCountDrift(DELETED, 4L);
+ metrics.setContainerCountDrift(CLOSING, 100L);
+ metrics.setContainerSyncDurationMs(DELETING, 200L);
+ metrics.setScmContainerSyncStatus(
+ ReconScmContainerSyncMetrics.SCM_CONTAINER_SYNC_STATUS_SUCCESS);
+ metrics.setScmContainerSyncDurationMs(500L);
+
+ MetricsRecordBuilder builder = getMetrics(metrics);
+
+
assertEquals(ReconScmContainerSyncMetrics.SCM_CONTAINER_SYNC_STATUS_SUCCESS,
+ getIntGauge("scmContainerSyncStatus", builder));
+ assertEquals(500L, getLongGauge("scmContainerSyncDurationMs", builder));
+ assertEquals(10L, getLongGauge("openContainerSyncDurationMs", builder));
+ assertEquals(20L,
+ getLongGauge("quasiClosedContainerSyncDurationMs", builder));
+ assertEquals(30L, getLongGauge("closedContainerSyncDurationMs", builder));
+ assertEquals(40L, getLongGauge("deletedContainerSyncDurationMs", builder));
+ assertEquals(2L, getLongGauge("openContainerCountDrift", builder));
+ assertEquals(0L,
+ getLongGauge("quasiClosedContainerCountDrift", builder));
+ assertEquals(-3L, getLongGauge("closedContainerCountDrift", builder));
+ assertEquals(4L, getLongGauge("deletedContainerCountDrift", builder));
+
+ verify(builder, never()).addGauge(
+ eqName(info("closingContainerSyncDurationMs", "")), eq(100L));
+ verify(builder, never()).addGauge(
+ eqName(info("deletingContainerSyncDurationMs", "")), eq(200L));
+ verify(builder, never()).addGauge(
+ eqName(info("closingContainerCountDrift", "")), eq(100L));
+ verify(builder, never()).addGauge(
+ eqName(info("deletingContainerCountDrift", "")), eq(200L));
+ }
+}
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconSCMContainerSyncIntegration.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconSCMContainerSyncIntegration.java
index acb751197f3..0d7147c7e8a 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconSCMContainerSyncIntegration.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconSCMContainerSyncIntegration.java
@@ -51,7 +51,9 @@
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.ozone.recon.ReconServerConfigKeys;
+import org.apache.hadoop.ozone.recon.metrics.ReconScmContainerSyncMetrics;
import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
@@ -59,7 +61,7 @@
/**
* Integration tests for {@link ReconStorageContainerSyncHelper} and
- * {@link
ReconStorageContainerManagerFacade#triggerTargetedSCMContainerSync()}.
+ * {@link ReconStorageContainerManagerFacade#triggerSCMContainerSync()}.
*
* <p>Uses a <em>real</em> {@link ReconContainerManager} backed by RocksDB
* (from {@link AbstractReconContainerManagerTest}) and a mocked
@@ -86,13 +88,20 @@ public class TestReconSCMContainerSyncIntegration
extends AbstractReconContainerManagerTest {
private StorageContainerServiceProvider mockScm;
+ private ReconScmContainerSyncMetrics metrics;
private ReconStorageContainerSyncHelper syncHelper;
@BeforeEach
void setupSyncHelper() {
mockScm = mock(StorageContainerServiceProvider.class);
+ metrics = ReconScmContainerSyncMetrics.create();
syncHelper = new ReconStorageContainerSyncHelper(
- mockScm, getConf(), getContainerManager());
+ mockScm, getConf(), getContainerManager(), metrics);
+ }
+
+ @AfterEach
+ void tearDownSyncMetrics() {
+ metrics.unRegister();
}
//
---------------------------------------------------------------------------
@@ -278,7 +287,7 @@ void multiplePagesAllBatchesProcessed() throws Exception {
getConf().setLong(
ReconServerConfigKeys.OZONE_RECON_SCM_CONTAINER_ID_BATCH_SIZE, 3L);
ReconStorageContainerSyncHelper pagedHelper = new
ReconStorageContainerSyncHelper(
- mockScm, getConf(), getContainerManager());
+ mockScm, getConf(), getContainerManager(), metrics);
when(mockScm.getContainerCount(CLOSED)).thenReturn(7L);
// Page 1: IDs 1-3
@@ -411,7 +420,7 @@ void openSyncUsesCursorAndOnlyFetchesNewOpenContainers()
throws Exception {
getConf().setLong(
ReconServerConfigKeys.OZONE_RECON_SCM_CONTAINER_ID_BATCH_SIZE, 2L);
ReconStorageContainerSyncHelper pagedHelper = new
ReconStorageContainerSyncHelper(
- mockScm, getConf(), getContainerManager());
+ mockScm, getConf(), getContainerManager(), metrics);
when(mockScm.getContainerCount(OPEN)).thenReturn(2L, 1L, 0L);
when(mockScm.getListOfContainerIDs(
@@ -681,7 +690,7 @@ void batchSizeLimitsDeletedListPagePerCycle() throws
Exception {
seedRecon(400, 10, CLOSED);
getConf().setInt(OZONE_RECON_SCM_DELETED_CONTAINER_CHECK_BATCH_SIZE, 3);
ReconStorageContainerSyncHelper batchHelper = new
ReconStorageContainerSyncHelper(
- mockScm, getConf(), getContainerManager());
+ mockScm, getConf(), getContainerManager(), metrics);
// SCM's DELETED list page 1 (IDs 400-402), then empty.
List<ContainerID> firstPage = idRange(400, 403);
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconStorageContainerManagerFacade.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconStorageContainerManagerFacade.java
index 54b01d3f5fb..42fd25b36c7 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconStorageContainerManagerFacade.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconStorageContainerManagerFacade.java
@@ -22,13 +22,17 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.File;
+import java.lang.reflect.Field;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
@@ -36,10 +40,12 @@
import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.recon.ReconTestInjector;
+import org.apache.hadoop.ozone.recon.metrics.ReconScmContainerSyncMetrics;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl;
import
org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -51,6 +57,15 @@ class TestReconStorageContainerManagerFacade {
@TempDir
private Path temporaryFolder;
+ private final List<ReconScmContainerSyncMetrics> metricsToUnregister =
+ new ArrayList<>();
+
+ @AfterEach
+ void tearDown() {
+ metricsToUnregister.forEach(ReconScmContainerSyncMetrics::unRegister);
+ metricsToUnregister.clear();
+ }
+
@Test
void testScmSnapshotDbIsOpenedAtCanonicalReconPath() throws Exception {
StorageContainerServiceProvider scmServiceProvider =
@@ -75,6 +90,7 @@ void testScmSnapshotDbIsOpenedAtCanonicalReconPath() throws
Exception {
ReconStorageContainerManagerFacade reconScm =
injector.getInstance(ReconStorageContainerManagerFacade.class);
+ metricsToUnregister.add(getContainerSyncMetrics(reconScm));
OzoneConfiguration conf = injector.getInstance(OzoneConfiguration.class);
File checkpointDir =
temporaryFolder.resolve("scm.snapshot.db_test").toFile();
@@ -98,4 +114,105 @@ void testScmSnapshotDbIsOpenedAtCanonicalReconPath()
throws Exception {
assertTrue(canonicalReconScmDb.exists());
assertFalse(checkpointDir.exists());
}
+
+ @Test
+ void testTriggerSCMContainerSyncSetsSuccessStatusAndDuration()
+ throws Exception {
+ ReconStorageContainerManagerFacade reconScm = createReconScmFacade();
+ ReconStorageContainerSyncHelper syncHelper =
+ mock(ReconStorageContainerSyncHelper.class);
+ ReconScmContainerSyncMetrics metrics = getContainerSyncMetrics(reconScm);
+ metrics.setScmContainerSyncDurationMs(-1L);
+ setContainerSyncHelper(reconScm, syncHelper);
+ when(syncHelper.syncWithSCMContainerInfo()).thenReturn(true);
+
+ assertTrue(reconScm.triggerSCMContainerSync());
+
+
assertEquals(ReconScmContainerSyncMetrics.SCM_CONTAINER_SYNC_STATUS_SUCCESS,
+ metrics.getScmContainerSyncStatus());
+ assertNotEquals(-1L, metrics.getScmContainerSyncDurationMs());
+ }
+
+ @Test
+ void testTriggerSCMContainerSyncSetsFailureStatusAndDuration()
+ throws Exception {
+ ReconStorageContainerManagerFacade reconScm = createReconScmFacade();
+ ReconStorageContainerSyncHelper syncHelper =
+ mock(ReconStorageContainerSyncHelper.class);
+ ReconScmContainerSyncMetrics metrics = getContainerSyncMetrics(reconScm);
+ metrics.setScmContainerSyncDurationMs(-1L);
+ setContainerSyncHelper(reconScm, syncHelper);
+ when(syncHelper.syncWithSCMContainerInfo()).thenReturn(false);
+
+ assertFalse(reconScm.triggerSCMContainerSync());
+
+
assertEquals(ReconScmContainerSyncMetrics.SCM_CONTAINER_SYNC_STATUS_FAILURE,
+ metrics.getScmContainerSyncStatus());
+ assertNotEquals(-1L, metrics.getScmContainerSyncDurationMs());
+ }
+
+ @Test
+ void testTriggerSCMContainerSyncSetsFailureStatusAndDurationOnException()
+ throws Exception {
+ ReconStorageContainerManagerFacade reconScm = createReconScmFacade();
+ ReconStorageContainerSyncHelper syncHelper =
+ mock(ReconStorageContainerSyncHelper.class);
+ ReconScmContainerSyncMetrics metrics = getContainerSyncMetrics(reconScm);
+ RuntimeException failure = new RuntimeException("sync failed");
+ metrics.setScmContainerSyncDurationMs(-1L);
+ setContainerSyncHelper(reconScm, syncHelper);
+ when(syncHelper.syncWithSCMContainerInfo()).thenThrow(failure);
+
+ RuntimeException thrown = assertThrows(RuntimeException.class,
+ reconScm::triggerSCMContainerSync);
+
+ assertEquals(failure, thrown);
+
assertEquals(ReconScmContainerSyncMetrics.SCM_CONTAINER_SYNC_STATUS_FAILURE,
+ metrics.getScmContainerSyncStatus());
+ assertNotEquals(-1L, metrics.getScmContainerSyncDurationMs());
+ }
+
+ private ReconStorageContainerManagerFacade createReconScmFacade()
+ throws Exception {
+ StorageContainerServiceProvider scmServiceProvider =
+ mock(StorageContainerServiceProviderImpl.class);
+ OMMetadataManager omMetadataManager = initializeNewOmMetadataManager(
+ Files.createDirectory(temporaryFolder.resolve("OmMetadata")).toFile());
+ ReconOMMetadataManager reconOMMetadataManager =
+ getTestReconOmMetadataManager(omMetadataManager,
+ Files.createDirectory(temporaryFolder.resolve("ReconOmMetadata"))
+ .toFile());
+ ReconTestInjector injector =
+ new ReconTestInjector.Builder(temporaryFolder.toFile())
+ .withReconSqlDb()
+ .withReconOm(reconOMMetadataManager)
+ .withOmServiceProvider(mock(OzoneManagerServiceProviderImpl.class))
+ .addBinding(OzoneStorageContainerManager.class,
+ ReconStorageContainerManagerFacade.class)
+ .withContainerDB()
+ .addBinding(StorageContainerServiceProvider.class,
+ scmServiceProvider)
+ .build();
+ ReconStorageContainerManagerFacade reconScm =
+ injector.getInstance(ReconStorageContainerManagerFacade.class);
+ metricsToUnregister.add(getContainerSyncMetrics(reconScm));
+ return reconScm;
+ }
+
+ private static void setContainerSyncHelper(
+ ReconStorageContainerManagerFacade reconScm,
+ ReconStorageContainerSyncHelper syncHelper) throws Exception {
+ Field helperField = ReconStorageContainerManagerFacade.class
+ .getDeclaredField("containerSyncHelper");
+ helperField.setAccessible(true);
+ helperField.set(reconScm, syncHelper);
+ }
+
+ private static ReconScmContainerSyncMetrics getContainerSyncMetrics(
+ ReconStorageContainerManagerFacade reconScm) throws Exception {
+ Field metricsField = ReconStorageContainerManagerFacade.class
+ .getDeclaredField("containerSyncMetrics");
+ metricsField.setAccessible(true);
+ return (ReconScmContainerSyncMetrics) metricsField.get(reconScm);
+ }
}
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconStorageContainerSyncHelper.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconStorageContainerSyncHelper.java
index 03b771e7ed2..d858c554c73 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconStorageContainerSyncHelper.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconStorageContainerSyncHelper.java
@@ -18,8 +18,12 @@
package org.apache.hadoop.ozone.recon.scm;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSED;
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.DELETED;
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.OPEN;
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.QUASI_CLOSED;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SCM_CONTAINER_ID_BATCH_SIZE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.argThat;
@@ -36,7 +40,10 @@
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
+import org.apache.hadoop.ozone.recon.metrics.ReconScmContainerSyncMetrics;
import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
class TestReconStorageContainerSyncHelper {
@@ -47,16 +54,51 @@ class TestReconStorageContainerSyncHelper {
private final ReconContainerManager mockContainerManager =
mock(ReconContainerManager.class);
- private final ReconStorageContainerSyncHelper syncHelper;
+ private ReconScmContainerSyncMetrics metrics;
+ private ReconStorageContainerSyncHelper syncHelper;
- TestReconStorageContainerSyncHelper() {
+ @BeforeEach
+ void setUp() {
+ metrics = ReconScmContainerSyncMetrics.create();
syncHelper = new ReconStorageContainerSyncHelper(
mockScmServiceProvider,
new OzoneConfiguration(),
- mockContainerManager
+ mockContainerManager,
+ metrics
);
}
+ @AfterEach
+ void tearDown() {
+ metrics.unRegister();
+ }
+
+ @Test
+ void testContainerSyncMetricsTrackPreSyncDriftAndDuration() throws Exception
{
+ when(mockScmServiceProvider.getContainerCount(OPEN)).thenReturn(5L);
+
when(mockScmServiceProvider.getContainerCount(QUASI_CLOSED)).thenReturn(1L);
+ when(mockScmServiceProvider.getContainerCount(CLOSED)).thenReturn(8L);
+ when(mockScmServiceProvider.getContainerCount(DELETED)).thenReturn(9L);
+ when(mockContainerManager.getContainerStateCount(OPEN)).thenReturn(3);
+
when(mockContainerManager.getContainerStateCount(QUASI_CLOSED)).thenReturn(1);
+ when(mockContainerManager.getContainerStateCount(CLOSED)).thenReturn(10);
+ when(mockContainerManager.getContainerStateCount(DELETED)).thenReturn(7);
+ when(mockScmServiceProvider.getListOfContainerIDs(
+ any(), any(Integer.class), any())).thenReturn(Collections.emptyList());
+
+ boolean result = syncHelper.syncWithSCMContainerInfo();
+
+ assertTrue(result);
+ assertEquals(2L, metrics.getContainerCountDrift(OPEN));
+ assertEquals(0L, metrics.getContainerCountDrift(QUASI_CLOSED));
+ assertEquals(-2L, metrics.getContainerCountDrift(CLOSED));
+ assertEquals(2L, metrics.getContainerCountDrift(DELETED));
+ assertTrue(metrics.getContainerSyncDurationMs(OPEN) >= 0);
+ assertTrue(metrics.getContainerSyncDurationMs(QUASI_CLOSED) >= 0);
+ assertTrue(metrics.getContainerSyncDurationMs(CLOSED) >= 0);
+ assertTrue(metrics.getContainerSyncDurationMs(DELETED) >= 0);
+ }
+
@Test
void testContainerMissingFromReconIsAdded() throws Exception {
ContainerID cid = ContainerID.valueOf(42L);
@@ -92,7 +134,7 @@ void testContainerMissingFromReconIsAddedWhenMultiplePages()
throws Exception {
OzoneConfiguration pagedConf = new OzoneConfiguration();
pagedConf.setLong(OZONE_RECON_SCM_CONTAINER_ID_BATCH_SIZE, 2L);
ReconStorageContainerSyncHelper pagedHelper = new
ReconStorageContainerSyncHelper(
- mockScmServiceProvider, pagedConf, mockContainerManager);
+ mockScmServiceProvider, pagedConf, mockContainerManager, metrics);
// Page 1: containers 1 and 2 (both missing from Recon)
ContainerID cid1 = ContainerID.valueOf(1L);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
