This is an automated email from the ASF dual-hosted git repository.
swamirishi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 03816462c89 HDDS-13949. Move dbTxSequenceNumber from SnapshotInfo to
LocalDataYaml file (#9313)
03816462c89 is described below
commit 03816462c898bbeb4744dc52196b075c7fb3cd08
Author: Swaminathan Balachandran <[email protected]>
AuthorDate: Thu Nov 20 15:14:47 2025 -0500
HDDS-13949. Move dbTxSequenceNumber from SnapshotInfo to LocalDataYaml file
(#9313)
---
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 4 +--
.../hadoop/ozone/om/helpers/SnapshotInfo.java | 27 +------------------
.../ozone/om/helpers/TestOmSnapshotInfo.java | 5 ----
.../hadoop/ozone/freon/TestOMSnapshotDAG.java | 3 ++-
.../src/main/proto/OmClientProtocol.proto | 2 +-
.../hadoop/ozone/om/OmSnapshotLocalData.java | 11 +++++++-
.../hadoop/ozone/om/OmSnapshotLocalDataYaml.java | 3 ++-
.../request/snapshot/OMSnapshotCreateRequest.java | 7 -----
.../om/snapshot/OmSnapshotLocalDataManager.java | 17 +++++++-----
.../om/snapshot/diff/delta/RDBDifferComputer.java | 8 +++---
.../ozone/om/TestOmSnapshotLocalDataYaml.java | 3 ++-
.../snapshot/TestOmSnapshotLocalDataManager.java | 30 ++++++++++++++--------
.../diff/delta/TestCompositeDeltaDiffComputer.java | 3 +--
.../snapshot/diff/delta/TestRDBDifferComputer.java | 12 +++------
15 files changed, 59 insertions(+), 77 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 9f7a1421537..42ca3f97b3b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -222,6 +222,7 @@ public final class OzoneConsts {
public static final String OM_SST_FILE_INFO_END_KEY = "endKey";
public static final String OM_SST_FILE_INFO_COL_FAMILY = "columnFamily";
public static final String OM_SLD_TXN_INFO = "transactionInfo";
+ public static final String OM_SLD_DB_TXN_SEQ_NUMBER = "dbTxSequenceNumber";
// YAML fields for .container files
public static final String CONTAINER_ID = "containerID";
diff --git
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
index e11abc3fdb1..7c4b878d62b 100644
---
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
+++
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
@@ -771,9 +771,7 @@ private Path getSSTFullPath(SstFileInfo sstFileInfo,
Path... dbPaths) throws IOE
* @param dest destination snapshot
* @param versionMap version map containing the connection between source
snapshot version and dest snapshot version.
* @param tablesToLookup tablesToLookup set of table (column family) names
used to restrict which SST files to return.
- * @return A list of SST files without extension.
- * e.g. ["/path/to/sstBackupDir/000050.sst",
- * "/path/to/sstBackupDir/000060.sst"]
+ * @return map of SST file absolute paths with extension to SstFileInfo.
*/
public synchronized Optional<Map<Path, SstFileInfo>>
getSSTDiffListWithFullPath(DifferSnapshotInfo src,
DifferSnapshotInfo dest, Map<Integer, Integer> versionMap,
TablePrefixInfo prefixInfo,
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
index 2e899aeca6e..0e2a4b6ee79 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
@@ -71,10 +71,6 @@ public final class SnapshotInfo implements Auditable,
CopyObject<SnapshotInfo> {
private UUID pathPreviousSnapshotId;
private UUID globalPreviousSnapshotId;
private String snapshotPath; // snapshot mask
- /**
- * RocksDB's transaction sequence number at the time of checkpoint creation.
- */
- private long dbTxSequenceNumber;
private boolean deepClean;
private boolean sstFiltered;
private long referencedSize;
@@ -98,7 +94,6 @@ private SnapshotInfo(Builder b) {
this.pathPreviousSnapshotId = b.pathPreviousSnapshotId;
this.globalPreviousSnapshotId = b.globalPreviousSnapshotId;
this.snapshotPath = b.snapshotPath;
- this.dbTxSequenceNumber = b.dbTxSequenceNumber;
this.deepClean = b.deepClean;
this.sstFiltered = b.sstFiltered;
this.referencedSize = b.referencedSize;
@@ -221,7 +216,6 @@ public SnapshotInfo.Builder toBuilder() {
.setPathPreviousSnapshotId(pathPreviousSnapshotId)
.setGlobalPreviousSnapshotId(globalPreviousSnapshotId)
.setSnapshotPath(snapshotPath)
- .setDbTxSequenceNumber(dbTxSequenceNumber)
.setDeepClean(deepClean)
.setSstFiltered(sstFiltered)
.setReferencedSize(referencedSize)
@@ -249,7 +243,6 @@ public static class Builder {
private UUID pathPreviousSnapshotId;
private UUID globalPreviousSnapshotId;
private String snapshotPath;
- private long dbTxSequenceNumber;
private boolean deepClean;
private boolean sstFiltered;
private long referencedSize;
@@ -327,12 +320,6 @@ public Builder setSnapshotPath(String snapshotPath) {
return this;
}
- /** @param dbTxSequenceNumber - RDB latest transaction sequence number. */
- public Builder setDbTxSequenceNumber(long dbTxSequenceNumber) {
- this.dbTxSequenceNumber = dbTxSequenceNumber;
- return this;
- }
-
/** @param deepClean - To be deep cleaned status for snapshot. */
public Builder setDeepClean(boolean deepClean) {
this.deepClean = deepClean;
@@ -441,7 +428,6 @@ public OzoneManagerProtocolProtos.SnapshotInfo
getProtobuf() {
}
sib.setSnapshotPath(snapshotPath)
- .setDbTxSequenceNumber(dbTxSequenceNumber)
.setDeepClean(deepClean);
return sib.build();
}
@@ -523,9 +509,7 @@ public static Builder builderFromProtobuf(
if (snapshotInfoProto.hasCreateTransactionInfo()) {
osib.setCreateTransactionInfo(snapshotInfoProto.getCreateTransactionInfo());
}
-
- osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath())
- .setDbTxSequenceNumber(snapshotInfoProto.getDbTxSequenceNumber());
+ osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath());
return osib;
}
@@ -568,14 +552,6 @@ public String getCheckpointDirName(int version) {
return getCheckpointDirName(getSnapshotId(), version);
}
- public long getDbTxSequenceNumber() {
- return dbTxSequenceNumber;
- }
-
- public void setDbTxSequenceNumber(long dbTxSequenceNumber) {
- this.dbTxSequenceNumber = dbTxSequenceNumber;
- }
-
/**
* Get the table key for this snapshot.
*/
@@ -761,7 +737,6 @@ public String toString() {
", pathPreviousSnapshotId: '" + pathPreviousSnapshotId + '\'' +
", globalPreviousSnapshotId: '" + globalPreviousSnapshotId + '\'' +
", snapshotPath: '" + snapshotPath + '\'' +
- ", dbTxSequenceNumber: '" + dbTxSequenceNumber + '\'' +
", deepClean: '" + deepClean + '\'' +
", sstFiltered: '" + sstFiltered + '\'' +
", referencedSize: '" + referencedSize + '\'' +
diff --git
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java
index e7695debd61..30f7e475c21 100644
---
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java
+++
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java
@@ -45,7 +45,6 @@ public class TestOmSnapshotInfo {
private static final UUID GLOBAL_PREVIOUS_SNAPSHOT_ID =
PATH_PREVIOUS_SNAPSHOT_ID;
private static final String SNAPSHOT_PATH = "test/path";
- private static final long DB_TX_SEQUENCE_NUMBER = 12345L;
private SnapshotInfo createSnapshotInfo() {
return new SnapshotInfo.Builder()
@@ -59,7 +58,6 @@ private SnapshotInfo createSnapshotInfo() {
.setPathPreviousSnapshotId(PATH_PREVIOUS_SNAPSHOT_ID)
.setGlobalPreviousSnapshotId(GLOBAL_PREVIOUS_SNAPSHOT_ID)
.setSnapshotPath(SNAPSHOT_PATH)
- .setDbTxSequenceNumber(DB_TX_SEQUENCE_NUMBER)
.setDeepClean(false)
.setSstFiltered(false)
.setReferencedSize(2000L)
@@ -84,7 +82,6 @@ private OzoneManagerProtocolProtos.SnapshotInfo
createSnapshotInfoProto() {
.setPathPreviousSnapshotID(toProtobuf(PATH_PREVIOUS_SNAPSHOT_ID))
.setGlobalPreviousSnapshotID(toProtobuf(GLOBAL_PREVIOUS_SNAPSHOT_ID))
.setSnapshotPath(SNAPSHOT_PATH)
- .setDbTxSequenceNumber(DB_TX_SEQUENCE_NUMBER)
.setDeepClean(false)
.setSstFiltered(false)
.setReferencedSize(2000L)
@@ -164,8 +161,6 @@ public void testSnapshotInfoProtoToSnapshotInfo() {
snapshotInfoActual.getBucketName());
assertEquals(snapshotInfoExpected.getSnapshotStatus(),
snapshotInfoActual.getSnapshotStatus());
- assertEquals(snapshotInfoExpected.getDbTxSequenceNumber(),
- snapshotInfoActual.getDbTxSequenceNumber());
assertEquals(snapshotInfoExpected.isDeepCleaned(),
snapshotInfoActual.isDeepCleaned());
assertEquals(snapshotInfoExpected.isSstFiltered(),
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java
index 3901eeeb0e4..c421e02705c 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java
@@ -161,7 +161,8 @@ private DifferSnapshotVersion getDifferSnapshotInfo(
throw new IllegalStateException(String.format("Duplicate key
%s", u));
}, TreeMap::new));
DifferSnapshotInfo dsi = new DifferSnapshotInfo((version) ->
Paths.get(checkpointPath),
- snapshotInfo.getSnapshotId(), snapshotInfo.getDbTxSequenceNumber(),
versionSstFiles);
+ snapshotInfo.getSnapshotId(),
snapshotLocalData.getSnapshotLocalData().getDbTxSequenceNumber(),
+ versionSstFiles);
return new DifferSnapshotVersion(dsi, 0,
COLUMN_FAMILIES_TO_TRACK_IN_DAG);
}
}
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 9d417700537..bdb3cc3cee3 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -881,7 +881,7 @@ message SnapshotInfo {
optional hadoop.hdds.UUID globalPreviousSnapshotID = 9;
optional string snapshotPath = 10;
optional string checkpointDir = 11 [deprecated = true];
- optional int64 dbTxSequenceNumber = 12;
+ optional int64 dbTxSequenceNumber = 12 [deprecated = true];
optional bool deepClean = 13;
optional bool sstFiltered = 14;
// snapshot reference size before any key replication or EC
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java
index 91ec8b673a8..f876a960601 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java
@@ -67,6 +67,9 @@ public class OmSnapshotLocalData implements
WithChecksum<OmSnapshotLocalData> {
// Stores the transactionInfo corresponding to OM when the snaphot is purged.
private TransactionInfo transactionInfo;
+ // Stores the rocksDB's transaction sequence number at the time of snapshot
creation.
+ private long dbTxSequenceNumber;
+
// Map of version to VersionMeta, using linkedHashMap since the order of the
map needs to be deterministic for
// checksum computation.
private final LinkedHashMap<Integer, VersionMeta> versionSstFileInfos;
@@ -78,7 +81,7 @@ public class OmSnapshotLocalData implements
WithChecksum<OmSnapshotLocalData> {
* Creates a OmSnapshotLocalData object with default values.
*/
public OmSnapshotLocalData(UUID snapshotId, List<LiveFileMetaData>
notDefraggedSSTFileList, UUID previousSnapshotId,
- TransactionInfo transactionInfo) {
+ TransactionInfo transactionInfo, long dbTxSequenceNumber) {
this.snapshotId = snapshotId;
this.isSSTFiltered = false;
this.lastDefragTime = 0L;
@@ -89,9 +92,14 @@ public OmSnapshotLocalData(UUID snapshotId,
List<LiveFileMetaData> notDefraggedS
this.version = 0;
this.previousSnapshotId = previousSnapshotId;
this.transactionInfo = transactionInfo;
+ this.dbTxSequenceNumber = dbTxSequenceNumber;
setChecksumTo0ByteArray();
}
+ public long getDbTxSequenceNumber() {
+ return dbTxSequenceNumber;
+ }
+
/**
* Copy constructor to create a deep copy of OmSnapshotLocalData object.
* @param source The source OmSnapshotLocalData to copy from
@@ -108,6 +116,7 @@ public OmSnapshotLocalData(OmSnapshotLocalData source) {
this.versionSstFileInfos = new LinkedHashMap<>();
setVersionSstFileInfos(source.versionSstFileInfos);
this.transactionInfo = source.transactionInfo;
+ this.dbTxSequenceNumber = source.dbTxSequenceNumber;
}
public TransactionInfo getTransactionInfo() {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java
index ad8046d719e..b72e74cf4a6 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java
@@ -172,9 +172,10 @@ public Object construct(Node node) {
final String prevSnapIdStr = (String)
nodes.get(OzoneConsts.OM_SLD_PREV_SNAP_ID);
UUID prevSnapId = prevSnapIdStr != null ?
UUID.fromString(prevSnapIdStr) : null;
final String purgeTxInfoStr = (String)
nodes.get(OzoneConsts.OM_SLD_TXN_INFO);
+ final long dbTxnSeqNumber =
((Number)nodes.get(OzoneConsts.OM_SLD_DB_TXN_SEQ_NUMBER)).longValue();
TransactionInfo transactionInfo = purgeTxInfoStr != null ?
TransactionInfo.valueOf(purgeTxInfoStr) : null;
OmSnapshotLocalData snapshotLocalData = new
OmSnapshotLocalData(snapId, Collections.emptyList(), prevSnapId,
- transactionInfo);
+ transactionInfo, dbTxnSeqNumber);
// Set version from YAML
Integer version = (Integer) nodes.get(OzoneConsts.OM_SLD_VERSION);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java
index 6211d411400..07a8aeed313 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java
@@ -32,7 +32,6 @@
import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.utils.TransactionInfo;
-import org.apache.hadoop.hdds.utils.db.RDBStore;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.OmUtils;
@@ -167,12 +166,6 @@ public OMClientResponse
validateAndUpdateCache(OzoneManager ozoneManager, Execut
throw new OMException("Snapshot already exists", FILE_ALREADY_EXISTS);
}
- // Note down RDB latest transaction sequence number, which is used
- // as snapshot generation in the Differ.
- final long dbLatestSequenceNumber =
- ((RDBStore) omMetadataManager.getStore()).getDb()
- .getLatestSequenceNumber();
- snapshotInfo.setDbTxSequenceNumber(dbLatestSequenceNumber);
ByteString txnBytes =
TransactionInfo.valueOf(context.getTermIndex()).toByteString();
snapshotInfo.setCreateTransactionInfo(txnBytes);
snapshotInfo.setLastTransactionInfo(txnBytes);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java
index be2a6706049..429c6776cab 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java
@@ -181,11 +181,14 @@ public String getSnapshotLocalPropertyYamlPath(UUID
snapshotId) {
public void createNewOmSnapshotLocalDataFile(RDBStore snapshotStore,
SnapshotInfo snapshotInfo) throws IOException {
try (WritableOmSnapshotLocalDataProvider snapshotLocalData =
new
WritableOmSnapshotLocalDataProvider(snapshotInfo.getSnapshotId(),
- () -> Pair.of(new
OmSnapshotLocalData(snapshotInfo.getSnapshotId(),
-
getLiveSSTFilesForCFs(snapshotStore.getDb().getManagedRocksDb(),
- COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT),
- snapshotInfo.getPathPreviousSnapshotId(), null),
- null))) {
+ () -> {
+ List<LiveFileMetaData> lfms =
getLiveSSTFilesForCFs(snapshotStore.getDb().getManagedRocksDb(),
+ COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT);
+ long dbTxnSeqNumber =
lfms.stream().mapToLong(LiveFileMetaData::largestSeqno).max().orElse(0L);
+ OmSnapshotLocalData localData = new
OmSnapshotLocalData(snapshotInfo.getSnapshotId(),
+ lfms, snapshotInfo.getPathPreviousSnapshotId(), null,
dbTxnSeqNumber);
+ return Pair.of(localData, null);
+ })) {
snapshotLocalData.commit();
}
}
@@ -263,16 +266,18 @@ private void addMissingSnapshotYamlFiles(
// Create a yaml file for snapshots which are missing
if (!snapshotLocalDataFile.exists()) {
List<LiveFileMetaData> sstList = Collections.emptyList();
+ long dbTxnSeqNumber = 0L;
if (snapshotInfo.getSnapshotStatus() == SNAPSHOT_ACTIVE) {
try (OmMetadataManagerImpl snapshotMetadataManager =
defaultSnapProvider.apply(snapshotInfo)) {
ManagedRocksDB snapDB =
((RDBStore)snapshotMetadataManager.getStore()).getDb().getManagedRocksDb();
sstList = getLiveSSTFilesForCFs(snapDB,
COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT);
+ dbTxnSeqNumber =
sstList.stream().mapToLong(LiveFileMetaData::largestSeqno).max().orElse(0L);
} catch (Exception e) {
throw new IOException(e);
}
}
OmSnapshotLocalData snapshotLocalData = new
OmSnapshotLocalData(snapshotId, sstList,
- snapshotInfo.getPathPreviousSnapshotId(), null);
+ snapshotInfo.getPathPreviousSnapshotId(), null, dbTxnSeqNumber);
// Set needsDefrag to true to indicate that the snapshot needs to be
defragmented, since the snapshot has
// never been defragmented before.
snapshotLocalData.setNeedsDefrag(true);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/RDBDifferComputer.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/RDBDifferComputer.java
index 0a59029fb0f..7ab7a5a68d7 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/RDBDifferComputer.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/RDBDifferComputer.java
@@ -73,9 +73,9 @@ public Optional<Map<Path, Pair<Path, SstFileInfo>>>
computeDeltaFiles(SnapshotIn
if (differ != null) {
try (OmSnapshotLocalDataManager.ReadableOmSnapshotLocalDataProvider
snapProvider =
getLocalDataProvider(toSnapshot.getSnapshotId(),
fromSnapshot.getSnapshotId())) {
- final DifferSnapshotInfo fromDSI =
getDSIFromSI(getActiveMetadataManager(), fromSnapshot,
+ final DifferSnapshotInfo fromDSI =
toDifferSnapshotInfo(getActiveMetadataManager(), fromSnapshot,
snapProvider.getPreviousSnapshotLocalData());
- final DifferSnapshotInfo toDSI =
getDSIFromSI(getActiveMetadataManager(), toSnapshot,
+ final DifferSnapshotInfo toDSI =
toDifferSnapshotInfo(getActiveMetadataManager(), toSnapshot,
snapProvider.getSnapshotLocalData());
final Map<Integer, Integer> versionMap =
snapProvider.getSnapshotLocalData().getVersionSstFileInfos().entrySet()
.stream().collect(toMap(Map.Entry::getKey, entry ->
entry.getValue().getPreviousSnapshotVersion()));
@@ -98,10 +98,10 @@ public Optional<Map<Path, Pair<Path, SstFileInfo>>>
computeDeltaFiles(SnapshotIn
/**
* Convert from SnapshotInfo to DifferSnapshotInfo.
*/
- private static DifferSnapshotInfo getDSIFromSI(OMMetadataManager
activeOmMetadataManager,
+ private static DifferSnapshotInfo toDifferSnapshotInfo(OMMetadataManager
activeOmMetadataManager,
SnapshotInfo snapshotInfo, OmSnapshotLocalData snapshotLocalData) throws
IOException {
final UUID snapshotId = snapshotInfo.getSnapshotId();
- final long dbTxSequenceNumber = snapshotInfo.getDbTxSequenceNumber();
+ final long dbTxSequenceNumber = snapshotLocalData.getDbTxSequenceNumber();
NavigableMap<Integer, List<SstFileInfo>> versionSstFiles =
snapshotLocalData.getVersionSstFileInfos().entrySet()
.stream().collect(toMap(Map.Entry::getKey,
entry -> entry.getValue().getSstFiles(), (u, v) -> {
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java
index 81f111e8464..34b9fbe397e 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java
@@ -118,7 +118,7 @@ private Pair<File, UUID> writeToYaml(UUID snapshotId,
String snapshotName, Trans
createLiveFileMetaData("sst2", "table1", "k3", "k4"),
createLiveFileMetaData("sst3", "table2", "k4", "k5"));
OmSnapshotLocalData dataYaml = new OmSnapshotLocalData(snapshotId,
notDefraggedSSTFileList,
- previousSnapshotId, transactionInfo);
+ previousSnapshotId, transactionInfo, 10);
// Set version
dataYaml.setVersion(42);
@@ -164,6 +164,7 @@ public void testWriteToYaml() throws IOException {
// Verify fields
assertEquals(44, snapshotData.getVersion());
+ assertEquals(10, snapshotData.getDbTxSequenceNumber());
assertTrue(snapshotData.getSstFiltered());
assertEquals(transactionInfo, snapshotData.getTransactionInfo());
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLocalDataManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLocalDataManager.java
index 6e804f330ae..76c013198e3 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLocalDataManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLocalDataManager.java
@@ -790,15 +790,17 @@ public void testCreateNewSnapshotLocalYaml() throws
IOException {
expNotDefraggedSSTFileList.put(DIRECTORY_TABLE, Stream.of("dt1",
"dt2").collect(Collectors.toList()));
List<LiveFileMetaData> mockedLiveFiles = new ArrayList<>();
+ int seqNumber = 0;
for (Map.Entry<String, List<String>> entry :
expNotDefraggedSSTFileList.entrySet()) {
String cfname = entry.getKey();
for (String fname : entry.getValue()) {
- mockedLiveFiles.add(createMockLiveFileMetaData("/" + fname + ".sst",
cfname, "k1", "k2"));
+ mockedLiveFiles.add(createMockLiveFileMetaData("/" + fname + ".sst",
cfname, "k1", "k2", seqNumber++));
}
}
+ int expectedDbTxSequenceNumber = seqNumber - 1;
// Add some other column families and files that should be ignored
- mockedLiveFiles.add(createMockLiveFileMetaData("ot1.sst", "otherTable",
"k1", "k2"));
- mockedLiveFiles.add(createMockLiveFileMetaData("ot2.sst", "otherTable",
"k1", "k2"));
+ mockedLiveFiles.add(createMockLiveFileMetaData("ot1.sst", "otherTable",
"k1", "k2", seqNumber++));
+ mockedLiveFiles.add(createMockLiveFileMetaData("ot2.sst", "otherTable",
"k1", "k2", seqNumber));
mockSnapshotStore(snapshotId, mockedLiveFiles);
localDataManager = getNewOmSnapshotLocalDataManager();
@@ -820,6 +822,7 @@ public void testCreateNewSnapshotLocalYaml() throws
IOException {
assertEquals(0L, localData.getLastDefragTime());
assertTrue(localData.getNeedsDefrag());
assertEquals(1, localData.getVersionSstFileInfos().size());
+ assertEquals(expectedDbTxSequenceNumber,
localData.getDbTxSequenceNumber());
}
@Test
@@ -829,12 +832,12 @@ public void testCreateNewOmSnapshotLocalDataFile() throws
IOException {
// Setup snapshot store mock
List<LiveFileMetaData> sstFiles = new ArrayList<>();
- sstFiles.add(createMockLiveFileMetaData("file1.sst", KEY_TABLE, "key1",
"key7"));
- sstFiles.add(createMockLiveFileMetaData("file2.sst", KEY_TABLE, "key3",
"key9"));
- sstFiles.add(createMockLiveFileMetaData("file3.sst", FILE_TABLE, "key1",
"key7"));
- sstFiles.add(createMockLiveFileMetaData("file4.sst", FILE_TABLE, "key1",
"key7"));
- sstFiles.add(createMockLiveFileMetaData("file5.sst", DIRECTORY_TABLE,
"key1", "key7"));
- sstFiles.add(createMockLiveFileMetaData("file6.sst", "colFamily1", "key1",
"key7"));
+ sstFiles.add(createMockLiveFileMetaData("file1.sst", KEY_TABLE, "key1",
"key7", 10));
+ sstFiles.add(createMockLiveFileMetaData("file2.sst", KEY_TABLE, "key3",
"key9", 20));
+ sstFiles.add(createMockLiveFileMetaData("file3.sst", FILE_TABLE, "key1",
"key7", 30));
+ sstFiles.add(createMockLiveFileMetaData("file4.sst", FILE_TABLE, "key1",
"key7", 100));
+ sstFiles.add(createMockLiveFileMetaData("file5.sst", DIRECTORY_TABLE,
"key1", "key7", 5000));
+ sstFiles.add(createMockLiveFileMetaData("file6.sst", "colFamily1", "key1",
"key7", 6000));
List<SstFileInfo> sstFileInfos = IntStream.range(0, sstFiles.size() - 1)
.mapToObj(sstFiles::get).map(lfm ->
new SstFileInfo(lfm.fileName().replace(".sst", ""),
@@ -856,6 +859,7 @@ public void testCreateNewOmSnapshotLocalDataFile() throws
IOException {
assertEquals(expectedVersionMeta, versionMeta);
// New Snapshot create needs to be defragged always.
assertTrue(snapshotLocalData.needsDefrag());
+ assertEquals(5000,
snapshotLocalData.getSnapshotLocalData().getDbTxSequenceNumber());
}
}
@@ -1062,11 +1066,17 @@ private SnapshotInfo createMockSnapshotInfo(UUID
snapshotId, UUID previousSnapsh
private LiveFileMetaData createMockLiveFileMetaData(String fileName, String
columnFamilyName, String smallestKey,
String largestKey) {
+ return createMockLiveFileMetaData(fileName, columnFamilyName, smallestKey,
largestKey, 0);
+ }
+
+ private LiveFileMetaData createMockLiveFileMetaData(String fileName, String
columnFamilyName, String smallestKey,
+ String largestKey, long largestSeqNumber) {
LiveFileMetaData liveFileMetaData = mock(LiveFileMetaData.class);
when(liveFileMetaData.columnFamilyName()).thenReturn(StringUtils.string2Bytes(columnFamilyName));
when(liveFileMetaData.fileName()).thenReturn(fileName);
when(liveFileMetaData.smallestKey()).thenReturn(StringUtils.string2Bytes(smallestKey));
when(liveFileMetaData.largestKey()).thenReturn(StringUtils.string2Bytes(largestKey));
+ when(liveFileMetaData.largestSeqno()).thenReturn(largestSeqNumber);
return liveFileMetaData;
}
@@ -1076,7 +1086,7 @@ private OmSnapshotLocalData createMockLocalData(UUID
snapshotId, UUID previousSn
sstFiles.add(createMockLiveFileMetaData("file2.sst", "columnFamily1",
"key3", "key10"));
sstFiles.add(createMockLiveFileMetaData("file3.sst", "columnFamily2",
"key1", "key8"));
sstFiles.add(createMockLiveFileMetaData("file4.sst", "columnFamily2",
"key0", "key10"));
- return new OmSnapshotLocalData(snapshotId, sstFiles, previousSnapshotId,
null);
+ return new OmSnapshotLocalData(snapshotId, sstFiles, previousSnapshotId,
null, 10);
}
private void createSnapshotLocalDataFile(UUID snapshotId, UUID
previousSnapshotId)
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestCompositeDeltaDiffComputer.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestCompositeDeltaDiffComputer.java
index e8af3f84dd7..b64520a05c1 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestCompositeDeltaDiffComputer.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestCompositeDeltaDiffComputer.java
@@ -718,8 +718,7 @@ private SnapshotInfo createMockSnapshotInfo(String
volumeName, String bucketName
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setName(snapshotName)
- .setSnapshotId(snapshotId)
- .setDbTxSequenceNumber(100L);
+ .setSnapshotId(snapshotId);
return builder.build();
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestRDBDifferComputer.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestRDBDifferComputer.java
index b4ba058a43c..19579a59e16 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestRDBDifferComputer.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestRDBDifferComputer.java
@@ -360,10 +360,10 @@ public void testComputeDeltaFilesWithVersionMapping()
throws IOException {
}
/**
- * Tests that getDSIFromSI throws exception when no versions found.
+ * Tests that toDifferSnapshotInfo throws exception when no versions found.
*/
@Test
- public void testGetDSIFromSIWithNoVersions() throws IOException {
+ public void testToDifferSnapshotInfoWithNoVersions() throws IOException {
rdbDifferComputer = new RDBDifferComputer(omSnapshotManager,
activeMetadataManager,
deltaDirPath, activityReporter);
@@ -486,8 +486,7 @@ private SnapshotInfo createMockSnapshotInfo(String
volumeName, String bucketName
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setName(snapshotName)
- .setSnapshotId(snapshotId)
- .setDbTxSequenceNumber(100L);
+ .setSnapshotId(snapshotId);
return builder.build();
}
@@ -528,8 +527,3 @@ private OmSnapshotLocalData
createMockSnapshotLocalDataWithVersions(UUID snapsho
return localData;
}
}
-
-
-
-
-
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
