This is an automated email from the ASF dual-hosted git repository.
swamirishi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 47473dc5a28 HDDS-13452. Prevent snapshot defrag from happening before
upgrade finalization (#9240)
47473dc5a28 is described below
commit 47473dc5a28c0c5a51498ca52bb072bbc974cfc6
Author: Swaminathan Balachandran <[email protected]>
AuthorDate: Thu Nov 13 11:27:09 2025 -0500
HDDS-13452. Prevent snapshot defrag from happening before upgrade
finalization (#9240)
---
.../hadoop/hdds/utils/MapBackedTableIterator.java | 6 +-
.../hadoop/hdds/utils/db/InMemoryTestTable.java | 13 +-
.../hdds/utils/db/StringInMemoryTestTable.java | 31 +++++
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 41 +++---
.../hadoop/ozone/om/SnapshotDefragService.java | 4 +-
.../om/snapshot/OmSnapshotLocalDataManager.java | 54 +++++++-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 3 +-
.../snapshot/TestOmSnapshotLocalDataManager.java | 140 ++++++++++++++++-----
8 files changed, 224 insertions(+), 68 deletions(-)
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/MapBackedTableIterator.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/MapBackedTableIterator.java
index 5af0e671d51..5ce574509da 100644
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/MapBackedTableIterator.java
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/MapBackedTableIterator.java
@@ -19,7 +19,7 @@
import java.util.Iterator;
import java.util.Map;
-import java.util.TreeMap;
+import java.util.NavigableMap;
import org.apache.hadoop.hdds.utils.db.Table;
/**
@@ -29,9 +29,9 @@ public class MapBackedTableIterator<V> implements
Table.KeyValueIterator<String,
private Iterator<Table.KeyValue<String, V>> itr;
private final String prefix;
- private final TreeMap<String, V> values;
+ private final NavigableMap<String, V> values;
- public MapBackedTableIterator(TreeMap<String, V> values, String prefix) {
+ public MapBackedTableIterator(NavigableMap<String, V> values, String prefix)
{
this.prefix = prefix;
this.values = values;
this.seekToFirst();
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/InMemoryTestTable.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/InMemoryTestTable.java
index 51baeb45177..34d23a039e8 100644
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/InMemoryTestTable.java
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/InMemoryTestTable.java
@@ -21,21 +21,22 @@
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.NavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
/**
* InMemory Table implementation for tests.
*/
-public final class InMemoryTestTable<KEY, VALUE> implements Table<KEY, VALUE> {
- private final Map<KEY, VALUE> map;
+public class InMemoryTestTable<KEY, VALUE> implements Table<KEY, VALUE> {
+ private final NavigableMap<KEY, VALUE> map;
public InMemoryTestTable() {
this(Collections.emptyMap());
}
public InMemoryTestTable(Map<KEY, VALUE> map) {
- this.map = new ConcurrentHashMap<>();
+ this.map = new ConcurrentSkipListMap<>(map);
this.map.putAll(map);
}
@@ -124,4 +125,8 @@ public void dumpToFileWithPrefix(File externalFile, KEY
prefix) {
public void loadFromFile(File externalFile) {
throw new UnsupportedOperationException();
}
+
+ NavigableMap<KEY, VALUE> getMap() {
+ return map;
+ }
}
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/StringInMemoryTestTable.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/StringInMemoryTestTable.java
new file mode 100644
index 00000000000..572a5ab69ea
--- /dev/null
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/StringInMemoryTestTable.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.utils.db;
+
+import org.apache.hadoop.hdds.utils.MapBackedTableIterator;
+
+/**
+ * In memory test table for String keys.
+ * @param <V> Value type.
+ */
+public class StringInMemoryTestTable<V> extends InMemoryTestTable<String, V> {
+ @Override
+ public KeyValueIterator<String, V> iterator(String prefix,
KeyValueIterator.Type type) {
+ return new MapBackedTableIterator<>(getMap(), prefix);
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
index 453eb3b3b07..db5ac4d6340 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
@@ -46,9 +46,6 @@
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_REPORT_MAX_PAGE_SIZE_DEFAULT;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT_DEFAULT;
-import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DIRECTORY_TABLE;
-import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
-import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_KEY_NAME;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TIMEOUT;
@@ -62,7 +59,6 @@
import com.google.common.cache.CacheLoader;
import com.google.common.cache.RemovalListener;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
import jakarta.annotation.Nonnull;
import java.io.File;
import java.io.IOException;
@@ -111,10 +107,10 @@
import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer;
+import org.apache.ratis.util.function.CheckedFunction;
import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle;
-import org.rocksdb.LiveFileMetaData;
import org.rocksdb.RocksDBException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -177,7 +173,7 @@ public final class OmSnapshotManager implements
AutoCloseable {
* families before compaction.
*/
public static final Set<String> COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT =
- ImmutableSet.of(KEY_TABLE, DIRECTORY_TABLE, FILE_TABLE);
+ RocksDBCheckpointDiffer.COLUMN_FAMILIES_TO_TRACK_IN_DAG;
private final long diffCleanupServiceInterval;
private final int maxOpenSstFilesInSnapshotDb;
@@ -198,12 +194,9 @@ public final class OmSnapshotManager implements
AutoCloseable {
public OmSnapshotManager(OzoneManager ozoneManager) throws IOException {
OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl)
ozoneManager.getMetadataManager();
- this.snapshotLocalDataManager = new
OmSnapshotLocalDataManager(ozoneManager.getMetadataManager(),
- omMetadataManager.getSnapshotChainManager(),
ozoneManager.getConfiguration());
boolean isFilesystemSnapshotEnabled =
ozoneManager.isFilesystemSnapshotEnabled();
LOG.info("Ozone filesystem snapshot feature is {}.",
isFilesystemSnapshotEnabled ? "enabled" : "disabled");
-
// Confirm that snapshot feature can be safely disabled.
// Throw unchecked exception if that is not the case.
if (!isFilesystemSnapshotEnabled &&
@@ -216,7 +209,6 @@ public OmSnapshotManager(OzoneManager ozoneManager) throws
IOException {
"Please set config ozone.filesystem.snapshot.enabled to true and " +
"try to start this Ozone Manager again.");
}
-
this.options = new ManagedDBOptions();
this.options.setCreateIfMissing(true);
this.columnFamilyOptions = new ManagedColumnFamilyOptions();
@@ -231,6 +223,12 @@ public OmSnapshotManager(OzoneManager ozoneManager) throws
IOException {
OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES,
OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT
);
+ CheckedFunction<SnapshotInfo, OmMetadataManagerImpl, IOException>
defaultSnapDBProvider = snapshotInfo ->
+ getSnapshotOmMetadataManager(snapshotInfo, 0,
maxOpenSstFilesInSnapshotDb,
+ ozoneManager.getConfiguration());
+ this.snapshotLocalDataManager = new
OmSnapshotLocalDataManager(ozoneManager.getMetadataManager(),
+ omMetadataManager.getSnapshotChainManager(),
ozoneManager.getVersionManager(), defaultSnapDBProvider,
+ ozoneManager.getConfiguration());
Preconditions.checkArgument(this.maxOpenSstFilesInSnapshotDb >= -1,
OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES + " value should be larger than or
equal to -1.");
@@ -238,7 +236,6 @@ public OmSnapshotManager(OzoneManager ozoneManager) throws
IOException {
ColumnFamilyHandle snapDiffReportCf;
ColumnFamilyHandle snapDiffPurgedJobCf;
String dbPath = getDbPath(ozoneManager.getConfiguration());
-
try {
// Add default CF
columnFamilyDescriptors.add(new ColumnFamilyDescriptor(
@@ -377,6 +374,12 @@ public boolean canDisableFsSnapshot(OMMetadataManager
ommm) {
return isSnapshotInfoTableEmpty;
}
+ private static OmMetadataManagerImpl
getSnapshotOmMetadataManager(SnapshotInfo snapshotInfo, int version,
+ int maxOpenSstFilesInSnapshotDb, OzoneConfiguration conf) throws
IOException {
+ return new OmMetadataManagerImpl(conf,
snapshotInfo.getCheckpointDirName(version),
+ maxOpenSstFilesInSnapshotDb);
+ }
+
private CacheLoader<UUID, OmSnapshot> createCacheLoader() {
return new CacheLoader<UUID, OmSnapshot>() {
@@ -429,9 +432,8 @@ public OmSnapshot load(@Nonnull UUID snapshotId) throws
IOException {
}
try
(OmSnapshotLocalDataManager.ReadableOmSnapshotLocalDataMetaProvider
snapshotLocalDataProvider =
snapshotLocalDataManager.getOmSnapshotLocalDataMeta(snapshotInfo)) {
- snapshotMetadataManager = new OmMetadataManagerImpl(conf,
-
snapshotInfo.getCheckpointDirName(snapshotLocalDataProvider.getMeta().getVersion()),
- maxOpenSstFilesInSnapshotDb);
+ snapshotMetadataManager =
getSnapshotOmMetadataManager(snapshotInfo,
+ snapshotLocalDataProvider.getMeta().getVersion(),
maxOpenSstFilesInSnapshotDb, conf);
}
} catch (IOException e) {
LOG.error("Failed to retrieve snapshot: {}", snapshotTableKey, e);
@@ -596,17 +598,6 @@ public SnapshotDiffCleanupService
getSnapshotDiffCleanupService() {
return snapshotDiffCleanupService;
}
- /**
- * Captures the list of SST files for keyTable, fileTable and directoryTable
in the DB.
- * @param store AOS or snapshot DB for not defragged or defragged snapshot
respectively.
- * @return a Map of (table, set of SST files corresponding to the table)
- */
- public static List<LiveFileMetaData> getSnapshotSSTFileList(RDBStore store)
throws IOException {
- return store.getDb().getLiveFilesMetaData().stream()
- .filter(lfm ->
COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT.contains(StringUtils.bytes2String(lfm.columnFamilyName())))
- .collect(Collectors.toList());
- }
-
// Get OmSnapshot if the keyName has ".snapshot" key indicator
@SuppressWarnings("unchecked")
public UncheckedAutoCloseableSupplier<IOmMetadataReader>
getActiveFsMetadataOrSnapshot(
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java
index 3eb1bfadf25..e72a91a2775 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java
@@ -45,6 +45,7 @@
import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock;
import org.apache.hadoop.ozone.om.snapshot.MultiSnapshotLocks;
import org.apache.hadoop.ozone.om.snapshot.OmSnapshotLocalDataManager;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -347,7 +348,8 @@ private boolean shouldRun() {
return false;
}
// The service only runs if current OM node is ready
- return running.get() && ozoneManager.isRunning();
+ return running.get() && ozoneManager.isRunning() &&
+
ozoneManager.getVersionManager().isAllowed(OMLayoutFeature.SNAPSHOT_DEFRAG);
}
public AtomicLong getSnapshotsDefraggedCount() {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java
index 0e665167283..10e296a2c85 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java
@@ -20,6 +20,10 @@
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_LOCAL_DATA_MANAGER_SERVICE_INTERVAL;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_LOCAL_DATA_MANAGER_SERVICE_INTERVAL_DEFAULT;
import static
org.apache.hadoop.ozone.om.OmSnapshotLocalDataYaml.YAML_FILE_EXTENSION;
+import static
org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static
org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE;
+import static
org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.SNAPSHOT_DEFRAG;
+import static org.apache.ozone.rocksdb.util.RdbUtil.getLiveSSTFilesForCFs;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.graph.GraphBuilder;
@@ -52,7 +56,10 @@
import org.apache.hadoop.hdds.utils.Scheduler;
import org.apache.hadoop.hdds.utils.TransactionInfo;
import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB;
import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.OmSnapshotLocalData;
import org.apache.hadoop.ozone.om.OmSnapshotLocalData.VersionMeta;
import org.apache.hadoop.ozone.om.OmSnapshotLocalDataYaml;
@@ -63,8 +70,10 @@
import org.apache.hadoop.ozone.om.lock.HierarchicalResourceLockManager;
import
org.apache.hadoop.ozone.om.lock.HierarchicalResourceLockManager.HierarchicalResourceLock;
import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager;
import org.apache.hadoop.ozone.util.ObjectSerializer;
import org.apache.hadoop.ozone.util.YamlSerializer;
+import org.apache.ratis.util.function.CheckedFunction;
import org.apache.ratis.util.function.CheckedSupplier;
import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
import org.rocksdb.LiveFileMetaData;
@@ -113,7 +122,8 @@ public class OmSnapshotLocalDataManager implements
AutoCloseable {
private volatile boolean closed;
public OmSnapshotLocalDataManager(OMMetadataManager omMetadataManager,
- SnapshotChainManager snapshotChainManager,
+ SnapshotChainManager snapshotChainManager, OMLayoutVersionManager
omLayoutVersionManager,
+ CheckedFunction<SnapshotInfo, OmMetadataManagerImpl, IOException>
defaultSnapProvider,
OzoneConfiguration configuration) throws IOException {
this.localDataGraph = GraphBuilder.directed().build();
this.omMetadataManager = omMetadataManager;
@@ -128,7 +138,7 @@ public void computeAndSetChecksum(Yaml yaml,
OmSnapshotLocalData data) throws IO
this.versionNodeMap = new ConcurrentHashMap<>();
this.fullLock = new ReentrantReadWriteLock();
this.internalLock = new ReentrantReadWriteLock();
- init(configuration, snapshotChainManager);
+ init(configuration, snapshotChainManager, omLayoutVersionManager,
defaultSnapProvider);
}
@VisibleForTesting
@@ -172,7 +182,8 @@ public void createNewOmSnapshotLocalDataFile(RDBStore
snapshotStore, SnapshotInf
try (WritableOmSnapshotLocalDataProvider snapshotLocalData =
new
WritableOmSnapshotLocalDataProvider(snapshotInfo.getSnapshotId(),
() -> Pair.of(new
OmSnapshotLocalData(snapshotInfo.getSnapshotId(),
-
OmSnapshotManager.getSnapshotSSTFileList(snapshotStore),
+
getLiveSSTFilesForCFs(snapshotStore.getDb().getManagedRocksDb(),
+ COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT),
snapshotInfo.getPathPreviousSnapshotId(), null),
null))) {
snapshotLocalData.commit();
@@ -242,6 +253,32 @@ private void addSnapshotVersionMeta(UUID snapshotId,
SnapshotVersionsMeta snapsh
}
}
+ private void addMissingSnapshotYamlFiles(
+ CheckedFunction<SnapshotInfo, OmMetadataManagerImpl, IOException>
defaultSnapProvider) throws IOException {
+ try (Table.KeyValueIterator<String, SnapshotInfo> itr =
omMetadataManager.getSnapshotInfoTable().iterator()) {
+ while (itr.hasNext()) {
+ SnapshotInfo snapshotInfo = itr.next().getValue();
+ UUID snapshotId = snapshotInfo.getSnapshotId();
+ File snapshotLocalDataFile = new
File(getSnapshotLocalPropertyYamlPath(snapshotId));
+ // Create a yaml file for snapshots which are missing
+ if (!snapshotLocalDataFile.exists()) {
+ List<LiveFileMetaData> sstList = Collections.emptyList();
+ if (snapshotInfo.getSnapshotStatus() == SNAPSHOT_ACTIVE) {
+ try (OmMetadataManagerImpl snapshotMetadataManager =
defaultSnapProvider.apply(snapshotInfo)) {
+ ManagedRocksDB snapDB =
((RDBStore)snapshotMetadataManager.getStore()).getDb().getManagedRocksDb();
+ sstList = getLiveSSTFilesForCFs(snapDB,
COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+ OmSnapshotLocalData snapshotLocalData = new
OmSnapshotLocalData(snapshotId, sstList,
+ snapshotInfo.getPathPreviousSnapshotId(), null);
+ snapshotLocalDataSerializer.save(snapshotLocalDataFile,
snapshotLocalData);
+ }
+ }
+ }
+ }
+
void addVersionNodeWithDependents(OmSnapshotLocalData snapshotLocalData)
throws IOException {
if (versionNodeMap.containsKey(snapshotLocalData.getSnapshotId())) {
return;
@@ -297,12 +334,18 @@ Map<UUID, Integer> getSnapshotToBeCheckedForOrphans() {
return snapshotToBeCheckedForOrphans;
}
- private void init(OzoneConfiguration configuration, SnapshotChainManager
chainManager) throws IOException {
+ private void init(OzoneConfiguration configuration, SnapshotChainManager
chainManager,
+ OMLayoutVersionManager layoutVersionManager,
+ CheckedFunction<SnapshotInfo, OmMetadataManagerImpl, IOException>
defaultSnapProvider) throws IOException {
this.locks = omMetadataManager.getHierarchicalLockManager();
this.snapshotToBeCheckedForOrphans = new ConcurrentHashMap<>();
RDBStore store = (RDBStore) omMetadataManager.getStore();
String checkpointPrefix = store.getDbLocation().getName();
File snapshotDir = new File(store.getSnapshotsParentDir());
+ boolean upgradeNeeded = !layoutVersionManager.isAllowed(SNAPSHOT_DEFRAG);
+ if (upgradeNeeded) {
+ addMissingSnapshotYamlFiles(defaultSnapProvider);
+ }
File[] localDataFiles = snapshotDir.listFiles(
(dir, name) -> name.startsWith(checkpointPrefix) &&
name.endsWith(YAML_FILE_EXTENSION));
if (localDataFiles == null) {
@@ -826,7 +869,8 @@ private SnapshotVersionsMeta
validateModification(OmSnapshotLocalData snapshotLo
}
public void addSnapshotVersion(RDBStore snapshotStore) throws IOException {
- List<LiveFileMetaData> sstFiles =
OmSnapshotManager.getSnapshotSSTFileList(snapshotStore);
+ List<LiveFileMetaData> sstFiles =
getLiveSSTFilesForCFs(snapshotStore.getDb().getManagedRocksDb(),
+ COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT);
OmSnapshotLocalData previousSnapshotLocalData =
getPreviousSnapshotLocalData();
this.getSnapshotLocalData().addVersionSSTFileInfos(sstFiles,
previousSnapshotLocalData == null ? 0 :
previousSnapshotLocalData.getVersion());
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
index 7deeef51161..057484c673e 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
@@ -45,7 +45,8 @@ public enum OMLayoutFeature implements LayoutFeature {
QUOTA(6, "Ozone quota re-calculate"),
HBASE_SUPPORT(7, "Full support of hsync, lease recovery and listOpenFiles
APIs for HBase"),
- DELEGATION_TOKEN_SYMMETRIC_SIGN(8, "Delegation token signed by symmetric
key");
+ DELEGATION_TOKEN_SYMMETRIC_SIGN(8, "Delegation token signed by symmetric
key"),
+ SNAPSHOT_DEFRAG(9, "Supporting defragmentation of snapshot");
/////////////////////////////// /////////////////////////////
// Example OM Layout Feature with Actions
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLocalDataManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLocalDataManager.java
index e3b5d343b36..fff40480364 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLocalDataManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLocalDataManager.java
@@ -24,6 +24,8 @@
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DIRECTORY_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static
org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE;
+import static
org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -32,6 +34,7 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.CALLS_REAL_METHODS;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
@@ -40,6 +43,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
@@ -70,7 +74,11 @@
import org.apache.hadoop.hdds.utils.db.RDBStore;
import org.apache.hadoop.hdds.utils.db.RocksDatabase;
import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.StringInMemoryTestTable;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB;
import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.OmSnapshotLocalData;
import org.apache.hadoop.ozone.om.OmSnapshotLocalDataYaml;
import org.apache.hadoop.ozone.om.OmSnapshotManager;
@@ -80,8 +88,12 @@
import
org.apache.hadoop.ozone.om.lock.HierarchicalResourceLockManager.HierarchicalResourceLock;
import
org.apache.hadoop.ozone.om.snapshot.OmSnapshotLocalDataManager.ReadableOmSnapshotLocalDataProvider;
import
org.apache.hadoop.ozone.om.snapshot.OmSnapshotLocalDataManager.WritableOmSnapshotLocalDataProvider;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager;
+import org.apache.hadoop.ozone.upgrade.LayoutFeature;
import org.apache.hadoop.ozone.util.YamlSerializer;
import org.apache.ozone.rocksdb.util.SstFileInfo;
+import org.apache.ratis.util.function.CheckedFunction;
import org.assertj.core.util.Lists;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
@@ -96,6 +108,7 @@
import org.mockito.MockedStatic;
import org.mockito.MockitoAnnotations;
import org.rocksdb.LiveFileMetaData;
+import org.rocksdb.RocksDB;
import org.yaml.snakeyaml.Yaml;
/**
@@ -124,6 +137,9 @@ public class TestOmSnapshotLocalDataManager {
@TempDir
private Path tempDir;
+ @Mock
+ private OMLayoutVersionManager layoutVersionManager;
+
private OmSnapshotLocalDataManager localDataManager;
private AutoCloseable mocks;
@@ -177,6 +193,7 @@ public void setUp() throws IOException {
purgedSnapshotIdMap.clear();
snapshotUtilMock.when(() -> OmSnapshotManager.isSnapshotPurged(any(),
any(), any(), any()))
.thenAnswer(i -> purgedSnapshotIdMap.getOrDefault(i.getArgument(2),
false));
+
when(layoutVersionManager.isAllowed(any(LayoutFeature.class))).thenReturn(true);
conf.setInt(OZONE_OM_SNAPSHOT_LOCAL_DATA_MANAGER_SERVICE_INTERVAL, -1);
}
@@ -242,6 +259,15 @@ private void mockLockManager() throws IOException {
});
}
+ private OmSnapshotLocalDataManager getNewOmSnapshotLocalDataManager(
+ CheckedFunction<SnapshotInfo, OmMetadataManagerImpl, IOException>
provider) throws IOException {
+ return new OmSnapshotLocalDataManager(omMetadataManager, null,
layoutVersionManager, provider, conf);
+ }
+
+ private OmSnapshotLocalDataManager getNewOmSnapshotLocalDataManager() throws
IOException {
+ return getNewOmSnapshotLocalDataManager(null);
+ }
+
private List<UUID> createSnapshotLocalData(OmSnapshotLocalDataManager
snapshotLocalDataManager,
int numberOfSnapshots) throws IOException {
SnapshotInfo previousSnapshotInfo = null;
@@ -277,11 +303,14 @@ private void mockSnapshotStore(UUID snapshotId,
List<LiveFileMetaData> sstFiles)
// Setup snapshot store mock
File snapshotDbLocation =
OmSnapshotManager.getSnapshotPath(omMetadataManager, snapshotId, 0).toFile();
assertTrue(snapshotDbLocation.exists() || snapshotDbLocation.mkdirs());
-
when(snapshotStore.getDbLocation()).thenReturn(snapshotDbLocation);
RocksDatabase rocksDatabase = mock(RocksDatabase.class);
when(snapshotStore.getDb()).thenReturn(rocksDatabase);
- when(rocksDatabase.getLiveFilesMetaData()).thenReturn(sstFiles);
+ ManagedRocksDB db = mock(ManagedRocksDB.class);
+ when(rocksDatabase.getManagedRocksDb()).thenReturn(db);
+ RocksDB rdb = mock(RocksDB.class);
+ when(db.get()).thenReturn(rdb);
+ when(rdb.getLiveFilesMetaData()).thenReturn(sstFiles);
}
/**
@@ -295,7 +324,7 @@ private void mockSnapshotStore(UUID snapshotId,
List<LiveFileMetaData> sstFiles)
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testLockOrderingAgainstAnotherSnapshot(boolean read) throws
IOException {
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
List<UUID> snapshotIds = new ArrayList<>();
snapshotIds.add(null);
snapshotIds.addAll(createSnapshotLocalData(localDataManager, 20));
@@ -347,7 +376,7 @@ public void testLockOrderingAgainstAnotherSnapshot(boolean
read) throws IOExcept
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testVersionLockResolution(boolean read) throws IOException {
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
List<UUID> snapshotIds = createSnapshotLocalData(localDataManager, 5);
for (int snapIdx = 0; snapIdx < snapshotIds.size(); snapIdx++) {
UUID snapId = snapshotIds.get(snapIdx);
@@ -385,7 +414,7 @@ public void testVersionLockResolution(boolean read) throws
IOException {
@Test
public void
testWriteVersionAdditionValidationWithoutPreviousSnapshotVersionExisting()
throws IOException {
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
List<UUID> snapshotIds = createSnapshotLocalData(localDataManager, 2);
UUID snapId = snapshotIds.get(1);
try (WritableOmSnapshotLocalDataProvider omSnapshotLocalDataProvider =
@@ -401,7 +430,7 @@ public void
testWriteVersionAdditionValidationWithoutPreviousSnapshotVersionExis
@Test
public void testUpdateTransactionInfo() throws IOException {
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
TransactionInfo transactionInfo =
TransactionInfo.valueOf(ThreadLocalRandom.current().nextLong(),
ThreadLocalRandom.current().nextLong());
UUID snapshotId = createSnapshotLocalData(localDataManager, 1).get(0);
@@ -420,7 +449,7 @@ public void testUpdateTransactionInfo() throws IOException {
@Test
public void testAddVersionFromRDB() throws IOException {
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
List<UUID> snapshotIds = createSnapshotLocalData(localDataManager, 2);
addVersionsToLocalData(localDataManager, snapshotIds.get(0),
ImmutableMap.of(4, 5, 6, 8));
UUID snapId = snapshotIds.get(1);
@@ -457,7 +486,7 @@ private void validateVersions(OmSnapshotLocalDataManager
snapshotLocalDataManage
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testOrphanVersionDeletionWithVersionDeletion(boolean
purgeSnapshot) throws IOException {
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
List<UUID> snapshotIds = createSnapshotLocalData(localDataManager, 3);
UUID firstSnapId = snapshotIds.get(0);
UUID secondSnapId = snapshotIds.get(1);
@@ -489,7 +518,7 @@ public void
testOrphanVersionDeletionWithVersionDeletion(boolean purgeSnapshot)
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testOrphanVersionDeletionWithChainUpdate(boolean purgeSnapshot)
throws IOException {
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
List<UUID> snapshotIds = createSnapshotLocalData(localDataManager, 3);
UUID firstSnapId = snapshotIds.get(0);
UUID secondSnapId = snapshotIds.get(1);
@@ -526,7 +555,7 @@ public void
testOrphanVersionDeletionWithChainUpdate(boolean purgeSnapshot) thro
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testWriteWithChainUpdate(boolean previousSnapshotExisting)
throws IOException {
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
List<UUID> snapshotIds = createSnapshotLocalData(localDataManager, 3 +
(previousSnapshotExisting ? 1 : 0));
int snapshotIdx = 1 + (previousSnapshotExisting ? 1 : 0);
for (UUID snapshotId : snapshotIds) {
@@ -578,7 +607,7 @@ public void testWriteWithChainUpdate(boolean
previousSnapshotExisting) throws IO
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testWriteVersionValidation(boolean nextVersionExisting) throws
IOException {
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
List<UUID> snapshotIds = createSnapshotLocalData(localDataManager, 3);
UUID prevSnapId = snapshotIds.get(0);
UUID snapId = snapshotIds.get(1);
@@ -652,7 +681,7 @@ private void
addVersionsToLocalData(OmSnapshotLocalDataManager snapshotLocalData
@ParameterizedTest
@ValueSource(ints = {1, 2, 3})
public void testNeedsDefrag(int previousVersion) throws IOException {
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
List<UUID> snapshotIds = createSnapshotLocalData(localDataManager, 2);
for (UUID snapshotId : snapshotIds) {
try (ReadableOmSnapshotLocalDataProvider snap =
localDataManager.getOmSnapshotLocalData(snapshotId)) {
@@ -672,7 +701,7 @@ public void testNeedsDefrag(int previousVersion) throws
IOException {
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testVersionResolution(boolean read) throws IOException {
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
List<UUID> snapshotIds = createSnapshotLocalData(localDataManager, 5);
List<Map<Integer, Integer>> versionMaps = Arrays.asList(
ImmutableMap.of(4, 1, 6, 3, 8, 9, 11, 15),
@@ -725,7 +754,7 @@ public void testVersionResolution(boolean read) throws
IOException {
@Test
public void testConstructor() throws IOException {
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
assertNotNull(localDataManager);
}
@@ -734,7 +763,7 @@ public void
testGetSnapshotLocalPropertyYamlPathWithSnapshotInfo() throws IOExce
UUID snapshotId = UUID.randomUUID();
SnapshotInfo snapshotInfo = createMockSnapshotInfo(snapshotId, null);
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
File yamlPath = new
File(localDataManager.getSnapshotLocalPropertyYamlPath(snapshotInfo));
assertNotNull(yamlPath);
@@ -772,7 +801,7 @@ public void testCreateNewSnapshotLocalYaml() throws
IOException {
mockedLiveFiles.add(createMockLiveFileMetaData("ot2.sst", "otherTable",
"k1", "k2"));
mockSnapshotStore(snapshotId, mockedLiveFiles);
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
Path snapshotYaml =
Paths.get(localDataManager.getSnapshotLocalPropertyYamlPath(snapshotInfo));
// Create an existing YAML file for the snapshot
assertTrue(snapshotYaml.toFile().createNewFile());
@@ -813,7 +842,7 @@ public void testCreateNewOmSnapshotLocalDataFile() throws
IOException {
bytes2String(lfm.largestKey()),
bytes2String(lfm.columnFamilyName()))).collect(Collectors.toList());
mockSnapshotStore(snapshotId, sstFiles);
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
localDataManager.createNewOmSnapshotLocalDataFile(snapshotStore,
snapshotInfo);
@@ -838,7 +867,7 @@ public void testGetOmSnapshotLocalDataWithSnapshotInfo()
throws IOException {
// Create and write snapshot local data file
OmSnapshotLocalData localData = createMockLocalData(snapshotId, null);
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
// Write the file manually for testing
Path yamlPath =
Paths.get(localDataManager.getSnapshotLocalPropertyYamlPath(snapshotInfo.getSnapshotId()));
@@ -859,7 +888,7 @@ public void
testGetOmSnapshotLocalDataWithMismatchedSnapshotId() throws IOExcept
// Create local data with wrong snapshot ID
OmSnapshotLocalData localData = createMockLocalData(wrongSnapshotId, null);
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
Path yamlPath =
Paths.get(localDataManager.getSnapshotLocalPropertyYamlPath(snapshotId));
writeLocalDataToFile(localData, yamlPath);
@@ -875,7 +904,7 @@ public void testGetOmSnapshotLocalDataWithFile() throws
IOException {
OmSnapshotLocalData localData = createMockLocalData(snapshotId, null);
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
Path yamlPath = tempDir.resolve("test-snapshot.yaml");
writeLocalDataToFile(localData, yamlPath);
@@ -893,7 +922,7 @@ public void testAddVersionNodeWithDependents() throws
IOException {
.sorted(Comparator.comparing(String::valueOf)).collect(Collectors.toList());
UUID snapshotId = versionIds.get(0);
UUID previousSnapshotId = versionIds.get(1);
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
// Create snapshot directory structure and files
createSnapshotLocalDataFile(snapshotId, previousSnapshotId);
createSnapshotLocalDataFile(previousSnapshotId, null);
@@ -909,7 +938,7 @@ public void testAddVersionNodeWithDependentsAlreadyExists()
throws IOException {
createSnapshotLocalDataFile(snapshotId, null);
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
OmSnapshotLocalData localData = createMockLocalData(snapshotId, null);
@@ -931,7 +960,7 @@ public void testInitWithExistingYamlFiles() throws
IOException {
createSnapshotLocalDataFile(snapshotId, previousSnapshotId);
// Initialize - should load existing files
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
assertNotNull(localDataManager);
Map<UUID, OmSnapshotLocalDataManager.SnapshotVersionsMeta> versionMap =
@@ -940,6 +969,56 @@ public void testInitWithExistingYamlFiles() throws
IOException {
assertEquals(versionMap.keySet(), new HashSet<>(versionIds));
}
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testInitWithMissingYamlFiles(boolean needsUpgrade) throws
IOException {
+ Table<String, SnapshotInfo> table = new StringInMemoryTestTable<>();
+ when(omMetadataManager.getSnapshotInfoTable()).thenReturn(table);
+ UUID snap3 = UUID.randomUUID();
+ UUID snap2 = UUID.randomUUID();
+ UUID snap1 = UUID.randomUUID();
+ CheckedFunction<SnapshotInfo, OmMetadataManagerImpl, IOException>
mockedProvider = (snapshotInfo) -> {
+ if (snapshotInfo.getSnapshotId().equals(snap2)) {
+ throw new IOException("SnapshotId should not be " + snap2 + " since it
is deleted");
+ }
+ mockSnapshotStore(snapshotInfo.getSnapshotId(),
ImmutableList.of(createMockLiveFileMetaData(
+ snapshotInfo.getSnapshotId() + ".sst", KEY_TABLE,
snapshotInfo.getSnapshotId() + "k1",
+ snapshotInfo.getSnapshotId() + "k2")));
+ OmMetadataManagerImpl snapshotMetadataManager =
mock(OmMetadataManagerImpl.class);
+ when(snapshotMetadataManager.getStore()).thenReturn(snapshotStore);
+ return snapshotMetadataManager;
+ };
+ table.put("snap3", createMockSnapshotInfo(snap3, null, SNAPSHOT_ACTIVE));
+ table.put("snap2", createMockSnapshotInfo(snap2, snap3, SNAPSHOT_DELETED));
+ table.put("snap1", createMockSnapshotInfo(snap1, snap2, SNAPSHOT_ACTIVE));
+
when(layoutVersionManager.isAllowed(eq(OMLayoutFeature.SNAPSHOT_DEFRAG))).thenReturn(!needsUpgrade);
+ localDataManager = getNewOmSnapshotLocalDataManager(mockedProvider);
+ if (needsUpgrade) {
+ assertEquals(ImmutableSet.of(snap1, snap2, snap3),
localDataManager.getVersionNodeMap().keySet());
+ Map<UUID, UUID> previousMap = ImmutableMap.of(snap2, snap3, snap1,
snap2);
+ Map<UUID, Map<Integer, OmSnapshotLocalData.VersionMeta>> expectedSstFile
= ImmutableMap.of(
+ snap3, ImmutableMap.of(0,
+ new OmSnapshotLocalData.VersionMeta(0, ImmutableList.of(
+ new SstFileInfo(snap3.toString(), snap3 + "k1", snap3 +
"k2", KEY_TABLE)))),
+ snap1, ImmutableMap.of(0,
+ new OmSnapshotLocalData.VersionMeta(0, ImmutableList.of(
+ new SstFileInfo(snap1.toString(), snap1 + "k1", snap1 +
"k2", KEY_TABLE)))),
+ snap2, ImmutableMap.of(0,
+ new OmSnapshotLocalData.VersionMeta(0, ImmutableList.of())));
+ for (UUID snapshotId : localDataManager.getVersionNodeMap().keySet()) {
+ try (ReadableOmSnapshotLocalDataProvider
readableOmSnapshotLocalDataProvider =
+ localDataManager.getOmSnapshotLocalData(snapshotId)) {
+ OmSnapshotLocalData snapshotLocalData =
readableOmSnapshotLocalDataProvider.getSnapshotLocalData();
+ assertEquals(snapshotId, snapshotLocalData.getSnapshotId());
+ assertEquals(previousMap.get(snapshotId),
snapshotLocalData.getPreviousSnapshotId());
+ assertEquals(expectedSstFile.get(snapshotId),
snapshotLocalData.getVersionSstFileInfos());
+ }
+ }
+ } else {
+ assertEquals(ImmutableSet.of(),
localDataManager.getVersionNodeMap().keySet());
+ }
+ }
+
@Test
public void testInitWithInvalidPathThrowsException() throws IOException {
UUID snapshotId = UUID.randomUUID();
@@ -950,14 +1029,12 @@ public void testInitWithInvalidPathThrowsException()
throws IOException {
writeLocalDataToFile(localData, wrongPath);
// Should throw IOException during init
- assertThrows(IOException.class, () -> {
- new OmSnapshotLocalDataManager(omMetadataManager, null, conf);
- });
+ assertThrows(IOException.class, this::getNewOmSnapshotLocalDataManager);
}
@Test
public void testClose() throws IOException {
- localDataManager = new OmSnapshotLocalDataManager(omMetadataManager, null,
conf);
+ localDataManager = getNewOmSnapshotLocalDataManager();
// Should not throw exception
localDataManager.close();
}
@@ -965,14 +1042,19 @@ public void testClose() throws IOException {
// Helper methods
private SnapshotInfo createMockSnapshotInfo(UUID snapshotId, UUID
previousSnapshotId) {
+ return createMockSnapshotInfo(snapshotId, previousSnapshotId, null);
+ }
+
+ private SnapshotInfo createMockSnapshotInfo(UUID snapshotId, UUID
previousSnapshotId,
+ SnapshotInfo.SnapshotStatus snapshotStatus) {
SnapshotInfo.Builder builder = SnapshotInfo.newBuilder()
.setSnapshotId(snapshotId)
.setName("snapshot-" + snapshotId);
-
+ builder.setSnapshotStatus(snapshotStatus == null ? SNAPSHOT_ACTIVE :
snapshotStatus);
if (previousSnapshotId != null) {
builder.setPathPreviousSnapshotId(previousSnapshotId);
}
-
+
return builder.build();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
