sadanand48 commented on code in PR #4273:
URL: https://github.com/apache/ozone/pull/4273#discussion_r1122634006
##########
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java:
##########
@@ -603,6 +603,11 @@ public final class OzoneConfigKeys {
OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_DAG_DAEMON_RUN_INTERVAL_DEFAULT =
TimeUnit.HOURS.toMillis(1);
+ public static final String OZONE_OM_SNAPSHOT_USE_FULL_DIFF =
+ "ozone.om.snapshot.use.full.diff";
+
+ public static final boolean OZONE_OM_SNAPSHOT_USE_FULL_DIFF_DEFAULT = false;
Review Comment:
Yeah its a good idea. Thanks for filing this one.
##########
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDiffUtils.java:
##########
@@ -35,20 +53,56 @@ public static boolean isKeyWithPrefixPresent(String
prefixForColumnFamily,
}
public static String constructBucketKey(String keyName) {
- if (!keyName.startsWith(OzoneConsts.OM_KEY_PREFIX)) {
- keyName = OzoneConsts.OM_KEY_PREFIX.concat(keyName);
+ if (!keyName.startsWith(OM_KEY_PREFIX)) {
+ keyName = OM_KEY_PREFIX.concat(keyName);
}
- String[] elements = keyName.split(OzoneConsts.OM_KEY_PREFIX);
+ String[] elements = keyName.split(OM_KEY_PREFIX);
String volume = elements[1];
String bucket = elements[2];
StringBuilder builder =
- new StringBuilder().append(OzoneConsts.OM_KEY_PREFIX).append(volume);
+ new StringBuilder().append(OM_KEY_PREFIX).append(volume);
if (StringUtils.isNotBlank(bucket)) {
- builder.append(OzoneConsts.OM_KEY_PREFIX).append(bucket);
+ builder.append(OM_KEY_PREFIX).append(bucket);
}
return builder.toString();
}
+ public static void filterRelevantSstFiles(Set<String> inputFiles,
+ Map<String, String> tableToPrefixMap) throws RocksDBException {
+ for (Iterator<String> fileIterator =
+ inputFiles.iterator(); fileIterator.hasNext();) {
+ String filepath = fileIterator.next();
+ if (!RocksDiffUtils.doesSstFileContainKeyRange(filepath,
+ tableToPrefixMap)) {
+ fileIterator.remove();
+ }
+ }
+ }
+
+ public static boolean doesSstFileContainKeyRange(String filepath,
+ Map<String, String> tableToPrefixMap) throws RocksDBException {
+ try (ManagedSstFileReader sstFileReader = ManagedSstFileReader.managed(
+ new SstFileReader(new Options()))) {
+ sstFileReader.get().open(filepath);
+ TableProperties properties = sstFileReader.get().getTableProperties();
+ String tableName = new String(properties.getColumnFamilyName(), UTF_8);
+ if (tableToPrefixMap.containsKey(tableName)) {
+ String prefix = tableToPrefixMap.get(tableName) + OM_KEY_PREFIX;
+ try (ManagedSstFileReaderIterator iterator =
+ ManagedSstFileReaderIterator.managed(sstFileReader.get()
+ .newIterator(new ReadOptions()))) {
+ iterator.get().seek(prefix.getBytes(UTF_8));
+ String seekResultKey = new String(iterator.get().key(), UTF_8);
+ return seekResultKey.startsWith(prefix);
+ }
+ }
+ return false;
+ } catch (RocksDBException e) {
+ LOG.error("Failed to read SST File ", e);
+ throw e;
Review Comment:
Done.
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java:
##########
@@ -42,6 +42,7 @@
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
+import org.rocksdb.RocksDBException;
Review Comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
