swamirishi commented on code in PR #9268:
URL: https://github.com/apache/ozone/pull/9268#discussion_r2524940082
##########
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java:
##########
@@ -863,53 +816,109 @@ public synchronized Optional<List<String>>
getSSTDiffListWithFullPath(DifferSnap
* must be non-null.
* @return A list of SST files without extension. e.g. ["000050", "000060"]
*/
- public synchronized Optional<List<String>> getSSTDiffList(DifferSnapshotInfo
src,
- DifferSnapshotInfo dest, Set<String> tablesToLookup) {
+ public synchronized Optional<List<SstFileInfo>>
getSSTDiffList(DifferSnapshotVersion src,
+ DifferSnapshotVersion dest, TablePrefixInfo prefixInfo, Set<String>
tablesToLookup, boolean useCompactionDag) {
// TODO: Reject or swap if dest is taken after src, once snapshot chain
// integration is done.
- Set<String> srcSnapFiles = readRocksDBLiveFiles(src.getRocksDB(),
tablesToLookup);
- Set<String> destSnapFiles = readRocksDBLiveFiles(dest.getRocksDB(),
tablesToLookup);
-
- Set<String> fwdDAGSameFiles = new HashSet<>();
- Set<String> fwdDAGDifferentFiles = new HashSet<>();
-
- LOG.debug("Doing forward diff from src '{}' to dest '{}'",
- src.getDbPath(), dest.getDbPath());
- internalGetSSTDiffList(src, dest, srcSnapFiles, destSnapFiles,
- fwdDAGSameFiles, fwdDAGDifferentFiles);
+ Map<String, SstFileInfo> fwdDAGSameFiles = new HashMap<>();
+ Map<String, SstFileInfo> fwdDAGDifferentFiles = new HashMap<>();
+ if (useCompactionDag) {
+ LOG.debug("Doing forward diff from src '{}' to dest '{}'",
src.getDbPath(), dest.getDbPath());
+ internalGetSSTDiffList(src, dest, fwdDAGSameFiles, fwdDAGDifferentFiles);
+ } else {
+ Set<SstFileInfo> srcSstFileInfos = new
HashSet<>(src.getSstFileMap().values());
+ Set<SstFileInfo> destSstFileInfos = new
HashSet<>(src.getSstFileMap().values());
+ for (SstFileInfo srcSstFileInfo : srcSstFileInfos) {
+ if (destSstFileInfos.contains(srcSstFileInfo)) {
+ fwdDAGSameFiles.put(srcSstFileInfo.getFileName(), srcSstFileInfo);
+ } else {
+ fwdDAGDifferentFiles.put(srcSstFileInfo.getFileName(),
srcSstFileInfo);
+ }
+ }
+ for (SstFileInfo destSstFileInfo : destSstFileInfos) {
+ if (srcSstFileInfos.contains(destSstFileInfo)) {
+ fwdDAGSameFiles.put(destSstFileInfo.getFileName(), destSstFileInfo);
+ } else {
+ fwdDAGDifferentFiles.put(destSstFileInfo.getFileName(),
destSstFileInfo);
+ }
+ }
+ }
if (LOG.isDebugEnabled()) {
LOG.debug("Result of diff from src '" + src.getDbPath() + "' to dest '" +
dest.getDbPath() + "':");
StringBuilder logSB = new StringBuilder();
logSB.append("Fwd DAG same SST files: ");
- for (String file : fwdDAGSameFiles) {
+ for (String file : fwdDAGSameFiles.keySet()) {
logSB.append(file).append(SPACE_DELIMITER);
}
LOG.debug(logSB.toString());
logSB.setLength(0);
logSB.append("Fwd DAG different SST files: ");
- for (String file : fwdDAGDifferentFiles) {
+ for (String file : fwdDAGDifferentFiles.keySet()) {
logSB.append(file).append(SPACE_DELIMITER);
}
LOG.debug("{}", logSB);
}
// Check if the DAG traversal was able to reach all the destination SST
files.
- for (String destSnapFile : destSnapFiles) {
- if (!fwdDAGSameFiles.contains(destSnapFile) &&
!fwdDAGDifferentFiles.contains(destSnapFile)) {
+ for (String destSnapFile : dest.getSstFiles()) {
+ if (!fwdDAGSameFiles.containsKey(destSnapFile) &&
!fwdDAGDifferentFiles.containsKey(destSnapFile)) {
return Optional.empty();
}
}
- if (src.getTablePrefixes() != null && src.getTablePrefixes().size() != 0) {
- RocksDiffUtils.filterRelevantSstFiles(fwdDAGDifferentFiles,
src.getTablePrefixes(),
- compactionDag.getCompactionMap(), tablesToLookup, src.getRocksDB(),
dest.getRocksDB());
+ if (prefixInfo != null && prefixInfo.size() != 0) {
+ RocksDiffUtils.filterRelevantSstFiles(fwdDAGDifferentFiles,
tablesToLookup, prefixInfo);
+ }
+ return Optional.of(new ArrayList<>(fwdDAGDifferentFiles.values()));
+ }
+
+ /**
+ * This class represents a version of a snapshot in a database differ
operation.
+ * It contains metadata associated with a specific snapshot version,
including
+ * SST file information, generation id, and the database path for the given
version.
+ *
+ * Designed to work with `DifferSnapshotInfo`, this class allows the
retrieval of
+ * snapshot-related metadata and facilitates mapping of SST files for
version comparison
+ * and other operations.
+ *
+ * The core functionality is to store and provide read-only access to:
+ * - SST file information for a specified snapshot version.
+ * - Snapshot generation identifier.
+ * - Path to the database directory corresponding to the snapshot version.
+ */
+ public static class DifferSnapshotVersion {
+ private Map<String, SstFileInfo> sstFiles;
+ private long generation;
+ private Path dbPath;
+
+ public DifferSnapshotVersion(DifferSnapshotInfo differSnapshotInfo, int
version,
+ Set<String> tablesToLookup) {
+ this.sstFiles = differSnapshotInfo.getSstFiles(version, tablesToLookup)
+ .stream().collect(Collectors.toMap(SstFileInfo::getFileName,
identity()));
+ this.generation = differSnapshotInfo.getGeneration();
+ this.dbPath = differSnapshotInfo.getDbPath(version);
+ }
+
+ private Path getDbPath() {
+ return dbPath;
+ }
+
+ private long getGeneration() {
+ return generation;
+ }
+
+ private Set<String> getSstFiles() {
Review Comment:
This is not the sstFileName but just the id
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
