This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ea4b01b93b HDDS-7935. [Snapshot] LRU Cache entries may get
evicted/closed during long running processes (#4568)
ea4b01b93b is described below
commit ea4b01b93b0f0fba148193b64c70d7f1e34ec3cf
Author: Siyao Meng <[email protected]>
AuthorDate: Wed May 3 13:54:16 2023 -0700
HDDS-7935. [Snapshot] LRU Cache entries may get evicted/closed during long
running processes (#4568)
---
.../common/src/main/resources/ozone-default.xml | 16 ++-
.../hadoop/ozone/om/TestOmSnapshotFileSystem.java | 16 +--
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 13 +++
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 120 +++++++++------------
.../request/snapshot/OMSnapshotDeleteRequest.java | 5 +-
.../ozone/om/snapshot/SnapshotDiffManager.java | 6 ++
.../hadoop/ozone/om/snapshot/SnapshotUtils.java | 50 +++++++++
.../hadoop/ozone/om/TestOmSnapshotManager.java | 5 +
8 files changed, 143 insertions(+), 88 deletions(-)
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 97c06a900c..132fdb0a4c 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -3541,15 +3541,6 @@
However this parameter allows the namespace to support non-S3 compatible
characters.
</description>
</property>
- <property>
- <name>ozone.om.snapshot.cache.max.size</name>
- <value>10</value>
- <tag>OZONE, OM</tag>
- <description>
- Size of the OM Snapshot LRU cache. This is the maximum number of open
OM Snapshot RocksDb instances
- that will be held in memory at any time.
- </description>
- </property>
<property>
<name>ozone.om.container.location.cache.size</name>
@@ -3687,7 +3678,12 @@
<value>10</value>
<tag>OZONE, OM</tag>
<description>
- Maximum number of entries allowed in the snapshot cache.
+ Size of the OM Snapshot LRU cache.
+
+ This is a soft limit of open OM Snapshot RocksDB instances that will be
+ held. The actual number of cached instance could exceed this limit if
+ more than this number of snapshot instances are still in-use by snapDiff
+ or other tasks.
</description>
</property>
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java
index cac8181743..b5f62a67a0 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
@@ -59,6 +58,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.nio.ByteBuffer;
@@ -411,15 +411,19 @@ public class TestOmSnapshotFileSystem {
deleteSnapshot(snapshotName);
// Can't access keys in snapshot anymore with FS API. Should throw
exception
- final String errorMsg = "no longer active";
- LambdaTestUtils.intercept(OMException.class, errorMsg,
+ final String errorMsg1 = "no longer active";
+ LambdaTestUtils.intercept(FileNotFoundException.class, errorMsg1,
() -> o3fs.listStatus(snapshotRoot));
- LambdaTestUtils.intercept(OMException.class, errorMsg,
+ LambdaTestUtils.intercept(FileNotFoundException.class, errorMsg1,
() -> o3fs.listStatus(snapshotParent));
- LambdaTestUtils.intercept(OMException.class, errorMsg,
+ // Note: Different error message due to inconsistent FNFE client-side
+ // handling in BasicOzoneClientAdapterImpl#getFileStatus
+ // TODO: Reconciliation?
+ final String errorMsg2 = "No such file or directory";
+ LambdaTestUtils.intercept(FileNotFoundException.class, errorMsg2,
() -> o3fs.getFileStatus(snapshotKey1));
- LambdaTestUtils.intercept(OMException.class, errorMsg,
+ LambdaTestUtils.intercept(FileNotFoundException.class, errorMsg2,
() -> o3fs.getFileStatus(snapshotKey2));
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
index 655a13d51d..8d026326bb 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
@@ -260,6 +260,19 @@ public class OmSnapshot implements IOmMetadataReader,
Closeable {
omMetadataManager.getStore().close();
}
+ @Override
+ protected void finalize() throws Throwable {
+ // Verify that the DB handle has been closed, log warning otherwise
+ // https://softwareengineering.stackexchange.com/a/288724
+ if (!omMetadataManager.getStore().isClosed()) {
+ LOG.warn("{} is not closed properly. snapshotName: {}",
+ // Print hash code for debugging
+ omMetadataManager.getStore().toString(),
+ snapshotName);
+ }
+ super.finalize();
+ }
+
@VisibleForTesting
public OMMetadataManager getMetadataManager() {
return omMetadataManager;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
index 4b4dc50b82..9f1a2747f2 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
@@ -56,8 +56,6 @@ import
org.apache.hadoop.ozone.om.codec.OmDBDiffReportEntryCodec;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
-import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus;
-import org.apache.hadoop.ozone.om.service.SnapshotDeletingService;
import org.apache.hadoop.ozone.om.service.SnapshotDiffCleanupService;
import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffJob;
import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffManager;
@@ -89,8 +87,8 @@ import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_CLE
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DB_DIR;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_REPORT_MAX_PAGE_SIZE;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_REPORT_MAX_PAGE_SIZE_DEFAULT;
-import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_KEY_NAME;
+import static
org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotActive;
import static
org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.dropColumnFamilyHandle;
/**
@@ -157,6 +155,9 @@ public final class OmSnapshotManager implements
AutoCloseable {
private final int maxPageSize;
+ // Soft limit of the snapshot cache size.
+ private final int softCacheSize;
+
public OmSnapshotManager(OzoneManager ozoneManager) {
this.options = new ManagedDBOptions();
this.options.setCreateIfMissing(true);
@@ -205,29 +206,42 @@ public final class OmSnapshotManager implements
AutoCloseable {
.getStore()
.getRocksDBCheckpointDiffer();
- // size of lru cache
- int cacheSize = ozoneManager.getConfiguration().getInt(
+ // snapshot cache size, soft-limit
+ this.softCacheSize = ozoneManager.getConfiguration().getInt(
OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE,
OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE_DEFAULT);
CacheLoader<String, OmSnapshot> loader = createCacheLoader();
- RemovalListener<String, OmSnapshot> removalListener
- = notification -> {
- try {
- // close snapshot's rocksdb on eviction
- LOG.debug("Closing snapshot: {}", notification.getKey());
- // TODO: [SNAPSHOT] HDDS-7935.Close only when refcount reaches
zero?
- notification.getValue().close();
- } catch (IOException e) {
- LOG.error("Failed to close snapshot: {} {}",
- notification.getKey(), e);
- }
- };
+ RemovalListener<String, OmSnapshot> removalListener = notification -> {
+ try {
+ final String snapshotTableKey = notification.getKey();
+ final OmSnapshot omSnapshot = notification.getValue();
+ if (omSnapshot != null) {
+ // close snapshot's rocksdb on eviction
+ LOG.debug("Closing OmSnapshot '{}' due to {}",
+ snapshotTableKey, notification.getCause());
+ omSnapshot.close();
+ } else {
+ // Cache value would never be null in RemovalListener.
+
+ // When a soft reference is GC'ed by the JVM, this RemovalListener
+ // would be called. But the cache value should still exist at that
+ // point. Thus in-theory this condition won't be triggered by JVM GC
+ throw new IllegalStateException("Unexpected: OmSnapshot is null");
+ }
+ } catch (IOException e) {
+ LOG.error("Failed to close OmSnapshot: {}", notification.getKey(), e);
+ }
+ };
// init LRU cache
- snapshotCache = CacheBuilder.newBuilder()
- .maximumSize(cacheSize)
+ this.snapshotCache = CacheBuilder.newBuilder()
+ // Indicating OmSnapshot instances are softly referenced from the
cache.
+ // If no thread is holding a strong reference to an OmSnapshot instance
+ // (e.g. SnapDiff), the instance could be garbage collected by JVM at
+ // its discretion.
+ .softValues()
.removalListener(removalListener)
.build(loader);
@@ -266,31 +280,12 @@ public final class OmSnapshotManager implements
AutoCloseable {
@Nonnull
public OmSnapshot load(@Nonnull String snapshotTableKey)
throws IOException {
- SnapshotInfo snapshotInfo;
// see if the snapshot exists
- snapshotInfo = getSnapshotInfo(snapshotTableKey);
+ SnapshotInfo snapshotInfo = getSnapshotInfo(snapshotTableKey);
// Block snapshot from loading when it is no longer active e.g.
DELETED,
// unless this is called from SnapshotDeletingService.
- // TODO: [SNAPSHOT] However, snapshotCache.get() from other requests
- // (not from SDS) would be able to piggyback off of this because
- // snapshot still in cache won't trigger loader again.
- // This needs proper addressal in e.g. HDDS-7935
- // by introducing another cache just for SDS.
- // While the snapshotCache would host ACTIVE snapshots only.
- if (!snapshotInfo.getSnapshotStatus().equals(
- SnapshotStatus.SNAPSHOT_ACTIVE)) {
- if (isCalledFromSnapshotDeletingService()) {
- LOG.debug("Permitting {} to load snapshot {} in status: {}",
- SnapshotDeletingService.class.getSimpleName(),
- snapshotInfo.getTableKey(),
- snapshotInfo.getSnapshotStatus());
- } else {
- throw new OMException("Unable to load snapshot. " +
- "Snapshot with table key '" + snapshotTableKey +
- "' is no longer active", FILE_NOT_FOUND);
- }
- }
+ checkSnapshotActive(snapshotInfo);
CacheValue<SnapshotInfo> cacheValue =
ozoneManager.getMetadataManager().getSnapshotInfoTable()
@@ -309,7 +304,7 @@ public final class OmSnapshotManager implements
AutoCloseable {
snapshotMetadataManager = new OmMetadataManagerImpl(conf,
snapshotInfo.getCheckpointDirName(), isSnapshotInCache);
} catch (IOException e) {
- LOG.error("Failed to retrieve snapshot: {}, {}", snapshotTableKey,
e);
+ LOG.error("Failed to retrieve snapshot: {}", snapshotTableKey);
throw e;
}
@@ -343,24 +338,6 @@ public final class OmSnapshotManager implements
AutoCloseable {
return registry;
}
- /**
- * Helper method to check whether the loader is called from
- * SnapshotDeletingTask (return true) or not (return false).
- */
- private boolean isCalledFromSnapshotDeletingService() {
-
- StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
- for (StackTraceElement elem : stackTrace) {
- // Allow as long as loader is called from SDS. e.g. SnapshotDeletingTask
- if (elem.getClassName().startsWith(
- SnapshotDeletingService.class.getName())) {
- return true;
- }
- }
-
- return false;
- }
-
/**
* Get snapshot instance LRU cache.
* @return LoadingCache
@@ -450,7 +427,7 @@ public final class OmSnapshotManager implements
AutoCloseable {
try (TableIterator<String,
? extends Table.KeyValue<String, RepeatedOmKeyInfo>>
- keyIter = omMetadataManager.getDeletedTable().iterator()) {
+ keyIter = omMetadataManager.getDeletedTable().iterator()) {
keyIter.seek(beginKey);
// Continue only when there are entries of snapshot (bucket) scope
@@ -517,6 +494,15 @@ public final class OmSnapshotManager implements
AutoCloseable {
String snapshotTableKey = SnapshotInfo.getTableKey(volumeName,
bucketName, snapshotName);
+ // Block FS API reads when snapshot is not active.
+ checkSnapshotActive(ozoneManager, snapshotTableKey);
+
+ // Warn if actual cache size exceeds the soft limit already.
+ if (snapshotCache.size() > softCacheSize) {
+ LOG.warn("Snapshot cache size ({}) exceeds configured soft-limit
({}).",
+ snapshotCache.size(), softCacheSize);
+ }
+
// retrieve the snapshot from the cache
try {
return snapshotCache.get(snapshotTableKey);
@@ -538,7 +524,7 @@ public final class OmSnapshotManager implements
AutoCloseable {
}
public static String getSnapshotPath(OzoneConfiguration conf,
- SnapshotInfo snapshotInfo) {
+ SnapshotInfo snapshotInfo) {
return OMStorage.getOmDbDir(conf) +
OM_KEY_PREFIX + OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX +
OM_DB_NAME + snapshotInfo.getCheckpointDirName();
@@ -575,7 +561,7 @@ public final class OmSnapshotManager implements
AutoCloseable {
final OmSnapshot fs = snapshotCache.get(fsKey);
final OmSnapshot ts = snapshotCache.get(tsKey);
return snapshotDiffManager.getSnapshotDiffReport(volume, bucket, fs, ts,
- fsInfo, tsInfo, index, pageSize, forceFullDiff);
+ fsInfo, tsInfo, index, pageSize, forceFullDiff);
} catch (ExecutionException e) {
throw new IOException(e.getCause());
}
@@ -584,12 +570,10 @@ public final class OmSnapshotManager implements
AutoCloseable {
private void verifySnapshotInfoForSnapDiff(final SnapshotInfo fromSnapshot,
final SnapshotInfo toSnapshot)
throws IOException {
- if ((fromSnapshot.getSnapshotStatus() != SnapshotStatus.SNAPSHOT_ACTIVE) ||
- (toSnapshot.getSnapshotStatus() != SnapshotStatus.SNAPSHOT_ACTIVE)) {
- // TODO: [SNAPSHOT] Throw custom snapshot exception.
- throw new IOException("Cannot generate snapshot diff for non-active " +
- "snapshots.");
- }
+ // Block SnapDiff if either of the snapshots is not active.
+ checkSnapshotActive(fromSnapshot);
+ checkSnapshotActive(toSnapshot);
+ // Check snapshot creation time
if (fromSnapshot.getCreationTime() > toSnapshot.getCreationTime()) {
throw new IOException("fromSnapshot:" + fromSnapshot.getName() +
" should be older than to toSnapshot:" + toSnapshot.getName());
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java
index 7af3f1486f..eaaf63cf64 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java
@@ -184,10 +184,7 @@ public class OMSnapshotDeleteRequest extends
OMClientRequest {
omClientResponse = new OMSnapshotDeleteResponse(
omResponse.build(), tableKey, snapshotInfo);
- // Evict the snapshot entry from cache, and close the snapshot DB
- // Nothing happens if the key doesn't exist in cache (snapshot not
loaded)
- ozoneManager.getOmSnapshotManager().getSnapshotCache()
- .invalidate(tableKey);
+ // No longer need to invalidate the entry in the snapshot cache here.
} catch (IOException ex) {
exception = ex;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
index e01d465056..63e56d9f80 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
@@ -96,6 +96,7 @@ import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_THR
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF_DEFAULT;
import static org.apache.hadoop.ozone.om.OmSnapshotManager.DELIMITER;
+import static
org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotActive;
import static
org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.dropColumnFamilyHandle;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static
org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.getSnapshotInfo;
@@ -1210,6 +1211,11 @@ public class SnapshotDiffManager implements
AutoCloseable {
String fsKey = SnapshotInfo.getTableKey(volume, bucket, fromSnapshot);
String tsKey = SnapshotInfo.getTableKey(volume, bucket, toSnapshot);
+
+ // Block SnapDiff if either one of the snapshots is not active
+ checkSnapshotActive(ozoneManager, fsKey);
+ checkSnapshotActive(ozoneManager, tsKey);
+
try {
submitSnapDiffJob(jobKey, jobId, volume, bucket,
snapshotCache.get(fsKey),
snapshotCache.get(tsKey), fsInfo, tsInfo, forceFullDiff);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java
index a03f817de3..358f1ed1c0 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java
@@ -23,11 +23,14 @@ import
org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus;
+import org.apache.hadoop.ozone.om.service.SnapshotDeletingService;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.RocksDBException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
/**
@@ -83,4 +86,51 @@ public final class SnapshotUtils {
throw new RuntimeException(exception);
}
}
+
+
+ /**
+ * Throws OMException FILE_NOT_FOUND if snapshot is not in active status.
+ * @param snapshotTableKey snapshot table key
+ */
+ public static void checkSnapshotActive(OzoneManager ozoneManager,
+ String snapshotTableKey)
+ throws IOException {
+ checkSnapshotActive(getSnapshotInfo(ozoneManager, snapshotTableKey));
+ }
+
+ public static void checkSnapshotActive(SnapshotInfo snapInfo)
+ throws OMException {
+
+ if (snapInfo.getSnapshotStatus() != SnapshotStatus.SNAPSHOT_ACTIVE) {
+ if (isCalledFromSnapshotDeletingService()) {
+ LOG.debug("Permitting {} to load snapshot {} even in status: {}",
+ SnapshotDeletingService.class.getSimpleName(),
+ snapInfo.getTableKey(),
+ snapInfo.getSnapshotStatus());
+ } else {
+ throw new OMException("Unable to load snapshot. " +
+ "Snapshot with table key '" + snapInfo.getTableKey() +
+ "' is no longer active", FILE_NOT_FOUND);
+ }
+ }
+ }
+
+ /**
+ * Helper method to check whether the loader is called from
+ * SnapshotDeletingTask (return true) or not (return false).
+ */
+ private static boolean isCalledFromSnapshotDeletingService() {
+
+ StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
+ for (StackTraceElement elem : stackTrace) {
+ // Allow as long as loader is called from SDS. e.g. SnapshotDeletingTask
+ if (elem.getClassName().startsWith(
+ SnapshotDeletingService.class.getName())) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java
index c3162784b0..541d0a5c96 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java
@@ -122,6 +122,11 @@ public class TestOmSnapshotManager {
.checkForSnapshot(second.getVolumeName(),
second.getBucketName(), getSnapshotPrefix(second.getName()));
+ // As a workaround, invalidate all cache entries in order to trigger
+ // instances close in this test case, since JVM GC most likely would not
+ // have triggered and closed the instances yet at this point.
+ omSnapshotManager.getSnapshotCache().invalidateAll();
+
// confirm store was closed
verify(firstSnapshotStore, timeout(3000).times(1)).close();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
