This is an automated email from the ASF dual-hosted git repository.
swamirishi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 66ccc25280 HDDS-11908. Snapshot diff DAG traversal should not skip
node based on prefix presence (#7567)
66ccc25280 is described below
commit 66ccc25280cc5715e75b262c47774dbe8b25872a
Author: Swaminathan Balachandran <[email protected]>
AuthorDate: Fri Dec 13 05:39:24 2024 -0800
HDDS-11908. Snapshot diff DAG traversal should not skip node based on
prefix presence (#7567)
---
.../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 45 ----------------------
.../org/apache/ozone/rocksdiff/RocksDiffUtils.java | 33 ++++++++++++++++
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 4 +-
3 files changed, 35 insertions(+), 47 deletions(-)
diff --git
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
index de637c9326..548623f259 100644
---
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
+++
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
@@ -34,7 +34,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import com.google.protobuf.InvalidProtocolBufferException;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.StringUtils;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
@@ -944,10 +943,6 @@ public class RocksDBCheckpointDiffer implements
AutoCloseable,
Preconditions.checkArgument(sameFiles.isEmpty(), "Set must be empty");
Preconditions.checkArgument(differentFiles.isEmpty(), "Set must be empty");
- // Use source snapshot's table prefix. At this point Source and target's
- // table prefix should be same.
- Map<String, String> columnFamilyToPrefixMap = src.getTablePrefixes();
-
for (String fileName : srcSnapFiles) {
if (destSnapFiles.contains(fileName)) {
LOG.debug("Source '{}' and destination '{}' share the same SST '{}'",
@@ -1011,15 +1006,6 @@ public class RocksDBCheckpointDiffer implements
AutoCloseable,
}
for (CompactionNode nextNode : successors) {
- if (shouldSkipNode(nextNode, columnFamilyToPrefixMap)) {
- LOG.debug("Skipping next node: '{}' with startKey: '{}' and " +
- "endKey: '{}' because it doesn't have keys related to " +
- "columnFamilyToPrefixMap: '{}'.",
- nextNode.getFileName(), nextNode.getStartKey(),
- nextNode.getEndKey(), columnFamilyToPrefixMap);
- continue;
- }
-
if (sameFiles.contains(nextNode.getFileName()) ||
differentFiles.contains(nextNode.getFileName())) {
LOG.debug("Skipping known processed SST: {}",
@@ -1541,35 +1527,4 @@ public class RocksDBCheckpointDiffer implements
AutoCloseable,
return fileInfoBuilder.build();
}
- @VisibleForTesting
- boolean shouldSkipNode(CompactionNode node,
- Map<String, String> columnFamilyToPrefixMap) {
- // This is for backward compatibility. Before the compaction log table
- // migration, startKey, endKey and columnFamily information is not
persisted
- // in compaction log files.
- // Also for the scenario when there is an exception in reading SST files
- // for the file node.
- if (node.getStartKey() == null || node.getEndKey() == null ||
- node.getColumnFamily() == null) {
- LOG.debug("Compaction node with fileName: {} doesn't have startKey, " +
- "endKey and columnFamily details.", node.getFileName());
- return false;
- }
-
- if (MapUtils.isEmpty(columnFamilyToPrefixMap)) {
- LOG.debug("Provided columnFamilyToPrefixMap is null or empty.");
- return false;
- }
-
- if (!columnFamilyToPrefixMap.containsKey(node.getColumnFamily())) {
- LOG.debug("SstFile node: {} is for columnFamily: {} while filter map " +
- "contains columnFamilies: {}.", node.getFileName(),
- node.getColumnFamily(), columnFamilyToPrefixMap.keySet());
- return true;
- }
-
- String keyPrefix = columnFamilyToPrefixMap.get(node.getColumnFamily());
- return !RocksDiffUtils.isKeyWithPrefixPresent(keyPrefix,
node.getStartKey(),
- node.getEndKey());
- }
}
diff --git
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDiffUtils.java
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDiffUtils.java
index e116868410..576b932891 100644
---
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDiffUtils.java
+++
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDiffUtils.java
@@ -17,6 +17,8 @@
*/
package org.apache.ozone.rocksdiff;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions;
import org.apache.hadoop.hdds.utils.db.managed.ManagedReadOptions;
@@ -113,4 +115,35 @@ public final class RocksDiffUtils {
}
+ @VisibleForTesting
+ static boolean shouldSkipNode(CompactionNode node,
+ Map<String, String> columnFamilyToPrefixMap) {
+ // This is for backward compatibility. Before the compaction log table
+ // migration, startKey, endKey and columnFamily information is not
persisted
+ // in compaction log files.
+ // Also for the scenario when there is an exception in reading SST files
+ // for the file node.
+ if (node.getStartKey() == null || node.getEndKey() == null ||
+ node.getColumnFamily() == null) {
+ LOG.debug("Compaction node with fileName: {} doesn't have startKey, " +
+ "endKey and columnFamily details.", node.getFileName());
+ return false;
+ }
+
+ if (MapUtils.isEmpty(columnFamilyToPrefixMap)) {
+ LOG.debug("Provided columnFamilyToPrefixMap is null or empty.");
+ return false;
+ }
+
+ if (!columnFamilyToPrefixMap.containsKey(node.getColumnFamily())) {
+ LOG.debug("SstFile node: {} is for columnFamily: {} while filter map " +
+ "contains columnFamilies: {}.", node.getFileName(),
+ node.getColumnFamily(), columnFamilyToPrefixMap.keySet());
+ return true;
+ }
+
+ String keyPrefix = columnFamilyToPrefixMap.get(node.getColumnFamily());
+ return !isKeyWithPrefixPresent(keyPrefix, node.getStartKey(),
+ node.getEndKey());
+ }
}
diff --git
a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java
b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java
index 1e5302f249..438a0e74f2 100644
---
a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java
+++
b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java
@@ -1895,7 +1895,7 @@ public class TestRocksDBCheckpointDiffer {
.getCompactionNodeMap().values().stream()
.sorted(Comparator.comparing(CompactionNode::getFileName))
.map(node ->
- rocksDBCheckpointDiffer.shouldSkipNode(node,
+ RocksDiffUtils.shouldSkipNode(node,
columnFamilyToPrefixMap))
.collect(Collectors.toList());
@@ -1932,7 +1932,7 @@ public class TestRocksDBCheckpointDiffer {
rocksDBCheckpointDiffer.loadAllCompactionLogs();
- assertEquals(expectedResponse, rocksDBCheckpointDiffer.shouldSkipNode(node,
+ assertEquals(expectedResponse, RocksDiffUtils.shouldSkipNode(node,
columnFamilyToPrefixMap));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
