swamirishi commented on code in PR #9283:
URL: https://github.com/apache/ozone/pull/9283#discussion_r2539715696
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java:
##########
@@ -535,8 +536,7 @@ private static Path findLinkPath(Map<String, Map<Path,
Path>> files, Path file)
// Check if the files are hard linked to each other.
// Note comparison must be done against srcPath, because
// destPath may only exist on Follower.
- if (OmSnapshotUtils.getINode(srcPath).equals(
- OmSnapshotUtils.getINode(file))) {
+ if (getINode(srcPath).equals(getINode(file))) {
Review Comment:
Removed the visibleForTesting not relevant anymore
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/DeltaFileComputer.java:
##########
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot.diff.delta;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Set;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.ozone.rocksdb.util.SstFileInfo;
+
+/**
+ * The DeltaFileComputer interface defines a contract for computing delta files
+ * that represent changes between two snapshots. Implementations of this
+ * interface are responsible for determining the modifications made from a
+ * baseline snapshot to a target snapshot in the form of delta files.
+ */
+public interface DeltaFileComputer extends Closeable {
+
+ /**
+ * Retrieves the delta files representing changes between two snapshots for
specified tables.
+ *
+ * @param fromSnapshot the baseline snapshot from which changes are computed
+ * @param toSnapshot the target snapshot to which changes are compared
+ * @param tablesToLookup the set of table names to consider when determining
changes
+ * @return an {@code Optional} containing a collection of pairs, where each
pair consists of a
+ * {@code Path} representing the delta file and an associated {@code
SstFileInfo}, or
+ * an empty {@code Optional} if no changes are found
Review Comment:
done
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/FullDiffComputer.java:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot.diff.delta;
+
+import static org.apache.ozone.rocksdiff.RocksDiffUtils.filterRelevantSstFiles;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Consumer;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.TablePrefixInfo;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmSnapshot;
+import org.apache.hadoop.ozone.om.OmSnapshotManager;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus;
+import org.apache.ozone.rocksdb.util.RdbUtil;
+import org.apache.ozone.rocksdb.util.SstFileInfo;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * FullDiffComputer is a specialized implementation of
FileLinkDeltaFileComputer
+ * that computes the delta files between two snapshots. It identifies the
differences
+ * in files and generates corresponding links for easier processing of
snapshot diffs.
+ * This implementation handles cases of optimized inode-based comparisons as
well as
+ * fallback with full file list comparisons in case of exceptions.
+ */
+class FullDiffComputer extends FileLinkDeltaFileComputer {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(FullDiffComputer.class);
+
+ FullDiffComputer(OmSnapshotManager snapshotManager, OMMetadataManager
activeMetadataManager, Path deltaDirPath,
+ Consumer<SubStatus> activityReporter) throws IOException {
+ super(snapshotManager, activeMetadataManager, deltaDirPath,
activityReporter);
+ }
+
+ @Override
+ Optional<Map<Path, Pair<Path, SstFileInfo>>> computeDeltaFiles(SnapshotInfo
fromSnapshotInfo,
+ SnapshotInfo toSnapshotInfo, Set<String> tablesToLookup, TablePrefixInfo
tablePrefixInfo) throws IOException {
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> fromSnapHandle =
getSnapshot(fromSnapshotInfo);
+ UncheckedAutoCloseableSupplier<OmSnapshot> toSnapHandle =
getSnapshot(toSnapshotInfo)) {
+ OmSnapshot fromSnapshot = fromSnapHandle.get();
+ OmSnapshot toSnapshot = toSnapHandle.get();
+ Path fromSnapshotPath =
fromSnapshot.getMetadataManager().getStore().getDbLocation().getAbsoluteFile().toPath();
+ Path toSnapshotPath =
toSnapshot.getMetadataManager().getStore().getDbLocation().getAbsoluteFile().toPath();
Review Comment:
removed the annotation not relevant anymore
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/FullDiffComputer.java:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot.diff.delta;
+
+import static org.apache.ozone.rocksdiff.RocksDiffUtils.filterRelevantSstFiles;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Consumer;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.TablePrefixInfo;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmSnapshot;
+import org.apache.hadoop.ozone.om.OmSnapshotManager;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus;
+import org.apache.ozone.rocksdb.util.RdbUtil;
+import org.apache.ozone.rocksdb.util.SstFileInfo;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * FullDiffComputer is a specialized implementation of
FileLinkDeltaFileComputer
+ * that computes the delta files between two snapshots. It identifies the
differences
+ * in files and generates corresponding links for easier processing of
snapshot diffs.
+ * This implementation handles cases of optimized inode-based comparisons as
well as
+ * fallback with full file list comparisons in case of exceptions.
+ */
+class FullDiffComputer extends FileLinkDeltaFileComputer {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(FullDiffComputer.class);
+
+ FullDiffComputer(OmSnapshotManager snapshotManager, OMMetadataManager
activeMetadataManager, Path deltaDirPath,
+ Consumer<SubStatus> activityReporter) throws IOException {
+ super(snapshotManager, activeMetadataManager, deltaDirPath,
activityReporter);
+ }
+
+ @Override
+ Optional<Map<Path, Pair<Path, SstFileInfo>>> computeDeltaFiles(SnapshotInfo
fromSnapshotInfo,
+ SnapshotInfo toSnapshotInfo, Set<String> tablesToLookup, TablePrefixInfo
tablePrefixInfo) throws IOException {
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> fromSnapHandle =
getSnapshot(fromSnapshotInfo);
+ UncheckedAutoCloseableSupplier<OmSnapshot> toSnapHandle =
getSnapshot(toSnapshotInfo)) {
+ OmSnapshot fromSnapshot = fromSnapHandle.get();
+ OmSnapshot toSnapshot = toSnapHandle.get();
+ Path fromSnapshotPath =
fromSnapshot.getMetadataManager().getStore().getDbLocation().getAbsoluteFile().toPath();
+ Path toSnapshotPath =
toSnapshot.getMetadataManager().getStore().getDbLocation().getAbsoluteFile().toPath();
+ Map<Path, Pair<Path, SstFileInfo>> paths = new HashMap<>();
+ try {
+ Map<Object, SstFileInfo> fromSnapshotFiles =
getSSTFileMapForSnapshot(fromSnapshot, tablesToLookup,
+ tablePrefixInfo);
+ Map<Object, SstFileInfo> toSnapshotFiles =
getSSTFileMapForSnapshot(toSnapshot, tablesToLookup,
+ tablePrefixInfo);
+ for (Map.Entry<Object, SstFileInfo> entry :
fromSnapshotFiles.entrySet()) {
+ if (!toSnapshotFiles.containsKey(entry.getKey())) {
+ Path source = entry.getValue().getFilePath(fromSnapshotPath);
+ paths.put(source, Pair.of(createLink(source), entry.getValue()));
+ }
+ }
+ for (Map.Entry<Object, SstFileInfo> entry :
toSnapshotFiles.entrySet()) {
+ if (!fromSnapshotFiles.containsKey(entry.getKey())) {
+ Path source = entry.getValue().getFilePath(toSnapshotPath);
+ paths.put(source, Pair.of(createLink(source), entry.getValue()));
+ }
+ }
+ } catch (IOException e) {
+ // In case of exception during inode read use all files
+ LOG.error("Exception occurred while populating delta files for
snapDiff", e);
+ LOG.warn("Falling back to full file list comparison, inode-based
optimization skipped.");
+ paths.clear();
+ Set<SstFileInfo> fromSnapshotFiles =
getSSTFileSetForSnapshot(fromSnapshot, tablesToLookup, tablePrefixInfo);
+ Set<SstFileInfo> toSnapshotFiles =
getSSTFileSetForSnapshot(toSnapshot, tablesToLookup, tablePrefixInfo);
+ for (SstFileInfo sstFileInfo : fromSnapshotFiles) {
+ Path source = sstFileInfo.getFilePath(fromSnapshotPath);
+ paths.put(source, Pair.of(createLink(source), sstFileInfo));
+ }
+ for (SstFileInfo sstFileInfo : toSnapshotFiles) {
+ Path source = sstFileInfo.getFilePath(toSnapshotPath);
+ paths.put(source, Pair.of(createLink(source), sstFileInfo));
+ }
+ }
+ return Optional.of(paths);
+ }
+ }
+
+ static Map<Object, SstFileInfo> getSSTFileMapForSnapshot(OmSnapshot snapshot,
+ Set<String> tablesToLookUp, TablePrefixInfo tablePrefixInfo) throws
IOException {
+ return
filterRelevantSstFiles(RdbUtil.getSSTFilesWithInodesForComparison(((RDBStore)snapshot.getMetadataManager()
+ .getStore()).getDb().getManagedRocksDb(), tablesToLookUp),
tablesToLookUp, tablePrefixInfo);
+ }
+
+ static Set<SstFileInfo> getSSTFileSetForSnapshot(OmSnapshot snapshot,
Set<String> tablesToLookUp,
+ TablePrefixInfo tablePrefixInfo) {
+ return
filterRelevantSstFiles(RdbUtil.getSSTFilesForComparison(((RDBStore)snapshot.getMetadataManager().getStore())
+ .getDb().getManagedRocksDb(), tablesToLookUp), tablesToLookUp,
tablePrefixInfo);
Review Comment:
removed the annotation not relevant anymore
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/FullDiffComputer.java:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot.diff.delta;
+
+import static org.apache.ozone.rocksdiff.RocksDiffUtils.filterRelevantSstFiles;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Consumer;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.TablePrefixInfo;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmSnapshot;
+import org.apache.hadoop.ozone.om.OmSnapshotManager;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus;
+import org.apache.ozone.rocksdb.util.RdbUtil;
+import org.apache.ozone.rocksdb.util.SstFileInfo;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * FullDiffComputer is a specialized implementation of
FileLinkDeltaFileComputer
+ * that computes the delta files between two snapshots. It identifies the
differences
+ * in files and generates corresponding links for easier processing of
snapshot diffs.
+ * This implementation handles cases of optimized inode-based comparisons as
well as
+ * fallback with full file list comparisons in case of exceptions.
+ */
+class FullDiffComputer extends FileLinkDeltaFileComputer {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(FullDiffComputer.class);
+
+ FullDiffComputer(OmSnapshotManager snapshotManager, OMMetadataManager
activeMetadataManager, Path deltaDirPath,
+ Consumer<SubStatus> activityReporter) throws IOException {
+ super(snapshotManager, activeMetadataManager, deltaDirPath,
activityReporter);
+ }
+
+ @Override
+ Optional<Map<Path, Pair<Path, SstFileInfo>>> computeDeltaFiles(SnapshotInfo
fromSnapshotInfo,
+ SnapshotInfo toSnapshotInfo, Set<String> tablesToLookup, TablePrefixInfo
tablePrefixInfo) throws IOException {
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> fromSnapHandle =
getSnapshot(fromSnapshotInfo);
+ UncheckedAutoCloseableSupplier<OmSnapshot> toSnapHandle =
getSnapshot(toSnapshotInfo)) {
+ OmSnapshot fromSnapshot = fromSnapHandle.get();
+ OmSnapshot toSnapshot = toSnapHandle.get();
+ Path fromSnapshotPath =
fromSnapshot.getMetadataManager().getStore().getDbLocation().getAbsoluteFile().toPath();
+ Path toSnapshotPath =
toSnapshot.getMetadataManager().getStore().getDbLocation().getAbsoluteFile().toPath();
Review Comment:
removed the annotation not relevant anymore
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java:
##########
@@ -535,8 +536,7 @@ private static Path findLinkPath(Map<String, Map<Path,
Path>> files, Path file)
// Check if the files are hard linked to each other.
// Note comparison must be done against srcPath, because
// destPath may only exist on Follower.
- if (OmSnapshotUtils.getINode(srcPath).equals(
- OmSnapshotUtils.getINode(file))) {
+ if (getINode(srcPath).equals(getINode(file))) {
Review Comment:
Removed the visibleForTesting not relevant anymore
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/FullDiffComputer.java:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot.diff.delta;
+
+import static org.apache.ozone.rocksdiff.RocksDiffUtils.filterRelevantSstFiles;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Consumer;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.TablePrefixInfo;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmSnapshot;
+import org.apache.hadoop.ozone.om.OmSnapshotManager;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus;
+import org.apache.ozone.rocksdb.util.RdbUtil;
+import org.apache.ozone.rocksdb.util.SstFileInfo;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * FullDiffComputer is a specialized implementation of
FileLinkDeltaFileComputer
+ * that computes the delta files between two snapshots. It identifies the
differences
+ * in files and generates corresponding links for easier processing of
snapshot diffs.
+ * This implementation handles cases of optimized inode-based comparisons as
well as
+ * fallback with full file list comparisons in case of exceptions.
+ */
+class FullDiffComputer extends FileLinkDeltaFileComputer {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(FullDiffComputer.class);
+
+ FullDiffComputer(OmSnapshotManager snapshotManager, OMMetadataManager
activeMetadataManager, Path deltaDirPath,
+ Consumer<SubStatus> activityReporter) throws IOException {
+ super(snapshotManager, activeMetadataManager, deltaDirPath,
activityReporter);
+ }
+
+ @Override
+ Optional<Map<Path, Pair<Path, SstFileInfo>>> computeDeltaFiles(SnapshotInfo
fromSnapshotInfo,
+ SnapshotInfo toSnapshotInfo, Set<String> tablesToLookup, TablePrefixInfo
tablePrefixInfo) throws IOException {
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> fromSnapHandle =
getSnapshot(fromSnapshotInfo);
+ UncheckedAutoCloseableSupplier<OmSnapshot> toSnapHandle =
getSnapshot(toSnapshotInfo)) {
+ OmSnapshot fromSnapshot = fromSnapHandle.get();
+ OmSnapshot toSnapshot = toSnapHandle.get();
+ Path fromSnapshotPath =
fromSnapshot.getMetadataManager().getStore().getDbLocation().getAbsoluteFile().toPath();
+ Path toSnapshotPath =
toSnapshot.getMetadataManager().getStore().getDbLocation().getAbsoluteFile().toPath();
Review Comment:
removed the annotation not relevant anymore
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/FullDiffComputer.java:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot.diff.delta;
+
+import static org.apache.ozone.rocksdiff.RocksDiffUtils.filterRelevantSstFiles;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Consumer;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.TablePrefixInfo;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmSnapshot;
+import org.apache.hadoop.ozone.om.OmSnapshotManager;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus;
+import org.apache.ozone.rocksdb.util.RdbUtil;
+import org.apache.ozone.rocksdb.util.SstFileInfo;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * FullDiffComputer is a specialized implementation of
FileLinkDeltaFileComputer
+ * that computes the delta files between two snapshots. It identifies the
differences
+ * in files and generates corresponding links for easier processing of
snapshot diffs.
+ * This implementation handles cases of optimized inode-based comparisons as
well as
+ * fallback with full file list comparisons in case of exceptions.
+ */
+class FullDiffComputer extends FileLinkDeltaFileComputer {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(FullDiffComputer.class);
+
+ FullDiffComputer(OmSnapshotManager snapshotManager, OMMetadataManager
activeMetadataManager, Path deltaDirPath,
+ Consumer<SubStatus> activityReporter) throws IOException {
+ super(snapshotManager, activeMetadataManager, deltaDirPath,
activityReporter);
+ }
+
+ @Override
+ Optional<Map<Path, Pair<Path, SstFileInfo>>> computeDeltaFiles(SnapshotInfo
fromSnapshotInfo,
+ SnapshotInfo toSnapshotInfo, Set<String> tablesToLookup, TablePrefixInfo
tablePrefixInfo) throws IOException {
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> fromSnapHandle =
getSnapshot(fromSnapshotInfo);
+ UncheckedAutoCloseableSupplier<OmSnapshot> toSnapHandle =
getSnapshot(toSnapshotInfo)) {
+ OmSnapshot fromSnapshot = fromSnapHandle.get();
+ OmSnapshot toSnapshot = toSnapHandle.get();
+ Path fromSnapshotPath =
fromSnapshot.getMetadataManager().getStore().getDbLocation().getAbsoluteFile().toPath();
+ Path toSnapshotPath =
toSnapshot.getMetadataManager().getStore().getDbLocation().getAbsoluteFile().toPath();
Review Comment:
removed the annotation not relevant anymore
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/FullDiffComputer.java:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot.diff.delta;
+
+import static org.apache.ozone.rocksdiff.RocksDiffUtils.filterRelevantSstFiles;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Consumer;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.TablePrefixInfo;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmSnapshot;
+import org.apache.hadoop.ozone.om.OmSnapshotManager;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus;
+import org.apache.ozone.rocksdb.util.RdbUtil;
+import org.apache.ozone.rocksdb.util.SstFileInfo;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * FullDiffComputer is a specialized implementation of
FileLinkDeltaFileComputer
+ * that computes the delta files between two snapshots. It identifies the
differences
+ * in files and generates corresponding links for easier processing of
snapshot diffs.
+ * This implementation handles cases of optimized inode-based comparisons as
well as
+ * fallback with full file list comparisons in case of exceptions.
+ */
+class FullDiffComputer extends FileLinkDeltaFileComputer {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(FullDiffComputer.class);
+
+ FullDiffComputer(OmSnapshotManager snapshotManager, OMMetadataManager
activeMetadataManager, Path deltaDirPath,
+ Consumer<SubStatus> activityReporter) throws IOException {
+ super(snapshotManager, activeMetadataManager, deltaDirPath,
activityReporter);
+ }
+
+ @Override
+ Optional<Map<Path, Pair<Path, SstFileInfo>>> computeDeltaFiles(SnapshotInfo
fromSnapshotInfo,
+ SnapshotInfo toSnapshotInfo, Set<String> tablesToLookup, TablePrefixInfo
tablePrefixInfo) throws IOException {
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> fromSnapHandle =
getSnapshot(fromSnapshotInfo);
+ UncheckedAutoCloseableSupplier<OmSnapshot> toSnapHandle =
getSnapshot(toSnapshotInfo)) {
+ OmSnapshot fromSnapshot = fromSnapHandle.get();
+ OmSnapshot toSnapshot = toSnapHandle.get();
+ Path fromSnapshotPath =
fromSnapshot.getMetadataManager().getStore().getDbLocation().getAbsoluteFile().toPath();
+ Path toSnapshotPath =
toSnapshot.getMetadataManager().getStore().getDbLocation().getAbsoluteFile().toPath();
+ Map<Path, Pair<Path, SstFileInfo>> paths = new HashMap<>();
+ try {
+ Map<Object, SstFileInfo> fromSnapshotFiles =
getSSTFileMapForSnapshot(fromSnapshot, tablesToLookup,
+ tablePrefixInfo);
+ Map<Object, SstFileInfo> toSnapshotFiles =
getSSTFileMapForSnapshot(toSnapshot, tablesToLookup,
+ tablePrefixInfo);
+ for (Map.Entry<Object, SstFileInfo> entry :
fromSnapshotFiles.entrySet()) {
+ if (!toSnapshotFiles.containsKey(entry.getKey())) {
+ Path source = entry.getValue().getFilePath(fromSnapshotPath);
+ paths.put(source, Pair.of(createLink(source), entry.getValue()));
+ }
+ }
+ for (Map.Entry<Object, SstFileInfo> entry :
toSnapshotFiles.entrySet()) {
+ if (!fromSnapshotFiles.containsKey(entry.getKey())) {
+ Path source = entry.getValue().getFilePath(toSnapshotPath);
+ paths.put(source, Pair.of(createLink(source), entry.getValue()));
+ }
+ }
+ } catch (IOException e) {
+ // In case of exception during inode read use all files
+ LOG.error("Exception occurred while populating delta files for
snapDiff", e);
+ LOG.warn("Falling back to full file list comparison, inode-based
optimization skipped.");
+ paths.clear();
+ Set<SstFileInfo> fromSnapshotFiles =
getSSTFileSetForSnapshot(fromSnapshot, tablesToLookup, tablePrefixInfo);
+ Set<SstFileInfo> toSnapshotFiles =
getSSTFileSetForSnapshot(toSnapshot, tablesToLookup, tablePrefixInfo);
+ for (SstFileInfo sstFileInfo : fromSnapshotFiles) {
+ Path source = sstFileInfo.getFilePath(fromSnapshotPath);
+ paths.put(source, Pair.of(createLink(source), sstFileInfo));
+ }
+ for (SstFileInfo sstFileInfo : toSnapshotFiles) {
+ Path source = sstFileInfo.getFilePath(toSnapshotPath);
+ paths.put(source, Pair.of(createLink(source), sstFileInfo));
+ }
+ }
+ return Optional.of(paths);
+ }
+ }
+
+ static Map<Object, SstFileInfo> getSSTFileMapForSnapshot(OmSnapshot snapshot,
+ Set<String> tablesToLookUp, TablePrefixInfo tablePrefixInfo) throws
IOException {
+ return
filterRelevantSstFiles(RdbUtil.getSSTFilesWithInodesForComparison(((RDBStore)snapshot.getMetadataManager()
+ .getStore()).getDb().getManagedRocksDb(), tablesToLookUp),
tablesToLookUp, tablePrefixInfo);
+ }
+
+ static Set<SstFileInfo> getSSTFileSetForSnapshot(OmSnapshot snapshot,
Set<String> tablesToLookUp,
+ TablePrefixInfo tablePrefixInfo) {
+ return
filterRelevantSstFiles(RdbUtil.getSSTFilesForComparison(((RDBStore)snapshot.getMetadataManager().getStore())
+ .getDb().getManagedRocksDb(), tablesToLookUp), tablesToLookUp,
tablePrefixInfo);
Review Comment:
removed the annotation not relevant anymore
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/FileLinkDeltaFileComputer.java:
##########
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot.diff.delta;
+
+import static java.nio.file.Files.createDirectories;
+import static org.apache.commons.io.FilenameUtils.getExtension;
+import static org.apache.commons.io.file.PathUtils.deleteDirectory;
+
+import java.io.IOException;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.utils.db.TablePrefixInfo;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmSnapshot;
+import org.apache.hadoop.ozone.om.OmSnapshotManager;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import
org.apache.hadoop.ozone.om.snapshot.OmSnapshotLocalDataManager.ReadableOmSnapshotLocalDataProvider;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus;
+import org.apache.ozone.rocksdb.util.SstFileInfo;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The {@code FileLinkDeltaFileComputer} is an abstract class that provides a
+ * base implementation for the {@code DeltaFileComputer} interface. It is
+ * responsible for computing delta files by creating hard links to the
+ * relevant source files in a specified delta directory, enabling a compact
+ * representation of changes between snapshots.
+ *
+ * This class encapsulates the logic for managing snapshots and metadata,
+ * creating hard links for delta representation, and reporting activity
+ * during the computation process.
+ */
+public abstract class FileLinkDeltaFileComputer implements DeltaFileComputer {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(FileLinkDeltaFileComputer.class);
+ private final OmSnapshotManager omSnapshotManager;
+ private final OMMetadataManager activeMetadataManager;
+ private final Consumer<SubStatus> activityReporter;
+ private Path deltaDir;
+ private AtomicInteger linkFileCounter = new AtomicInteger(0);
+
+ FileLinkDeltaFileComputer(OmSnapshotManager snapshotManager,
OMMetadataManager activeMetadataManager,
+ Path deltaDirPath, Consumer<SubStatus> activityReporter) throws
IOException {
+ this.deltaDir = deltaDirPath.toAbsolutePath();
+ this.omSnapshotManager = snapshotManager;
+ this.activityReporter = activityReporter;
+ this.activeMetadataManager = activeMetadataManager;
+ createDirectories(deltaDir);
+ }
+
+ /**
+ * Computes the delta files between two snapshots based on the provided
parameters.
+ * The method determines the differences in data between the `fromSnapshot`
and
+ * `toSnapshot` and generates a mapping of paths to pairs consisting of a
resolved
+ * path and corresponding SST file information.
+ *
+ * @param fromSnapshot the source snapshot from which changes are calculated
+ * @param toSnapshot the target snapshot up to which changes are calculated
+ * @param tablesToLookup a set of table names to filter the tables that
should be considered
+ * @param tablePrefixInfo information about table prefixes to apply during
computation
+ * @return an Optional containing a map where the key is the delta file
path, and the value
+ * is a pair consisting of a resolved path and the corresponding SST
file information.
+ * Return empty if the delta files could not be computed.
+ * @throws IOException if an I/O error occurs during the computation process
+ */
+ abstract Optional<Map<Path, Pair<Path, SstFileInfo>>>
computeDeltaFiles(SnapshotInfo fromSnapshot,
+ SnapshotInfo toSnapshot, Set<String> tablesToLookup, TablePrefixInfo
tablePrefixInfo) throws IOException;
+
+ @Override
+ public final Collection<Pair<Path, SstFileInfo>> getDeltaFiles(SnapshotInfo
fromSnapshot,
+ SnapshotInfo toSnapshot, Set<String> tablesToLookup) throws IOException {
+ TablePrefixInfo tablePrefixInfo =
activeMetadataManager.getTableBucketPrefix(fromSnapshot.getVolumeName(),
+ fromSnapshot.getBucketName());
+ return computeDeltaFiles(fromSnapshot, toSnapshot, tablesToLookup,
+ tablePrefixInfo).map(Map::values).orElseThrow(() -> new
IOException(String.format(
+ "Failed to compute delta files for snapshots %s and %s
tablesToLookup : %s", fromSnapshot, toSnapshot,
+ tablesToLookup)));
+ }
+
+ void updateActivity(SubStatus status) {
+ activityReporter.accept(status);
+ }
+
+ Path createLink(Path path) throws IOException {
+ Path source = path.toAbsolutePath();
+ Path link;
+ boolean createdLink = false;
+ Path fileName = source.getFileName();
+ if (source.getFileName() == null) {
+ throw new IOException("Unable to create link for path " + source + "
since it has no file name");
+ }
+ String extension = getExtension(fileName.toString());
+ extension = StringUtils.isBlank(extension) ? "" : ("." + extension);
+ do {
+ link = deltaDir.resolve(linkFileCounter.incrementAndGet() + extension);
+ try {
+ Files.createLink(link, source);
+ createdLink = true;
+ } catch (FileAlreadyExistsException ignored) {
+ LOG.info("File for source {} already exists: at {}. Will attempt to
create link with a different path", source,
+ link);
+ }
+ } while (!createdLink);
Review Comment:
this shouldn't happen ideally. Not required
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/FullDiffComputer.java:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot.diff.delta;
+
+import static org.apache.ozone.rocksdiff.RocksDiffUtils.filterRelevantSstFiles;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Consumer;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.TablePrefixInfo;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmSnapshot;
+import org.apache.hadoop.ozone.om.OmSnapshotManager;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus;
+import org.apache.ozone.rocksdb.util.RdbUtil;
+import org.apache.ozone.rocksdb.util.SstFileInfo;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * FullDiffComputer is a specialized implementation of
FileLinkDeltaFileComputer
+ * that computes the delta files between two snapshots. It identifies the
differences
+ * in files and generates corresponding links for easier processing of
snapshot diffs.
+ * This implementation handles cases of optimized inode-based comparisons as
well as
+ * fallback with full file list comparisons in case of exceptions.
+ */
+class FullDiffComputer extends FileLinkDeltaFileComputer {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(FullDiffComputer.class);
+
+ FullDiffComputer(OmSnapshotManager snapshotManager, OMMetadataManager
activeMetadataManager, Path deltaDirPath,
+ Consumer<SubStatus> activityReporter) throws IOException {
+ super(snapshotManager, activeMetadataManager, deltaDirPath,
activityReporter);
+ }
+
+ @Override
+ Optional<Map<Path, Pair<Path, SstFileInfo>>> computeDeltaFiles(SnapshotInfo
fromSnapshotInfo,
+ SnapshotInfo toSnapshotInfo, Set<String> tablesToLookup, TablePrefixInfo
tablePrefixInfo) throws IOException {
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> fromSnapHandle =
getSnapshot(fromSnapshotInfo);
+ UncheckedAutoCloseableSupplier<OmSnapshot> toSnapHandle =
getSnapshot(toSnapshotInfo)) {
+ OmSnapshot fromSnapshot = fromSnapHandle.get();
+ OmSnapshot toSnapshot = toSnapHandle.get();
+ Path fromSnapshotPath =
fromSnapshot.getMetadataManager().getStore().getDbLocation().getAbsoluteFile().toPath();
+ Path toSnapshotPath =
toSnapshot.getMetadataManager().getStore().getDbLocation().getAbsoluteFile().toPath();
+ Map<Path, Pair<Path, SstFileInfo>> paths = new HashMap<>();
+ try {
+ Map<Object, SstFileInfo> fromSnapshotFiles =
getSSTFileMapForSnapshot(fromSnapshot, tablesToLookup,
+ tablePrefixInfo);
+ Map<Object, SstFileInfo> toSnapshotFiles =
getSSTFileMapForSnapshot(toSnapshot, tablesToLookup,
+ tablePrefixInfo);
+ for (Map.Entry<Object, SstFileInfo> entry :
fromSnapshotFiles.entrySet()) {
+ if (!toSnapshotFiles.containsKey(entry.getKey())) {
+ Path source = entry.getValue().getFilePath(fromSnapshotPath);
+ paths.put(source, Pair.of(createLink(source), entry.getValue()));
+ }
+ }
+ for (Map.Entry<Object, SstFileInfo> entry :
toSnapshotFiles.entrySet()) {
+ if (!fromSnapshotFiles.containsKey(entry.getKey())) {
+ Path source = entry.getValue().getFilePath(toSnapshotPath);
+ paths.put(source, Pair.of(createLink(source), entry.getValue()));
+ }
+ }
+ } catch (IOException e) {
+ // In case of exception during inode read use all files
+ LOG.error("Exception occurred while populating delta files for
snapDiff", e);
+ LOG.warn("Falling back to full file list comparison, inode-based
optimization skipped.");
+ paths.clear();
+ Set<SstFileInfo> fromSnapshotFiles =
getSSTFileSetForSnapshot(fromSnapshot, tablesToLookup, tablePrefixInfo);
+ Set<SstFileInfo> toSnapshotFiles =
getSSTFileSetForSnapshot(toSnapshot, tablesToLookup, tablePrefixInfo);
+ for (SstFileInfo sstFileInfo : fromSnapshotFiles) {
+ Path source = sstFileInfo.getFilePath(fromSnapshotPath);
+ paths.put(source, Pair.of(createLink(source), sstFileInfo));
+ }
+ for (SstFileInfo sstFileInfo : toSnapshotFiles) {
+ Path source = sstFileInfo.getFilePath(toSnapshotPath);
+ paths.put(source, Pair.of(createLink(source), sstFileInfo));
+ }
+ }
+ return Optional.of(paths);
+ }
+ }
+
+ static Map<Object, SstFileInfo> getSSTFileMapForSnapshot(OmSnapshot snapshot,
+ Set<String> tablesToLookUp, TablePrefixInfo tablePrefixInfo) throws
IOException {
+ return
filterRelevantSstFiles(RdbUtil.getSSTFilesWithInodesForComparison(((RDBStore)snapshot.getMetadataManager()
Review Comment:
removed the annotation not relevant anymore
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/FullDiffComputer.java:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot.diff.delta;
+
+import static org.apache.ozone.rocksdiff.RocksDiffUtils.filterRelevantSstFiles;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Consumer;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.TablePrefixInfo;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmSnapshot;
+import org.apache.hadoop.ozone.om.OmSnapshotManager;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus;
+import org.apache.ozone.rocksdb.util.RdbUtil;
+import org.apache.ozone.rocksdb.util.SstFileInfo;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * FullDiffComputer is a specialized implementation of
FileLinkDeltaFileComputer
+ * that computes the delta files between two snapshots. It identifies the
differences
+ * in files and generates corresponding links for easier processing of
snapshot diffs.
+ * This implementation handles cases of optimized inode-based comparisons as
well as
+ * fallback with full file list comparisons in case of exceptions.
+ */
+class FullDiffComputer extends FileLinkDeltaFileComputer {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(FullDiffComputer.class);
+
+ FullDiffComputer(OmSnapshotManager snapshotManager, OMMetadataManager
activeMetadataManager, Path deltaDirPath,
+ Consumer<SubStatus> activityReporter) throws IOException {
+ super(snapshotManager, activeMetadataManager, deltaDirPath,
activityReporter);
+ }
+
+ @Override
+ Optional<Map<Path, Pair<Path, SstFileInfo>>> computeDeltaFiles(SnapshotInfo
fromSnapshotInfo,
+ SnapshotInfo toSnapshotInfo, Set<String> tablesToLookup, TablePrefixInfo
tablePrefixInfo) throws IOException {
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> fromSnapHandle =
getSnapshot(fromSnapshotInfo);
+ UncheckedAutoCloseableSupplier<OmSnapshot> toSnapHandle =
getSnapshot(toSnapshotInfo)) {
+ OmSnapshot fromSnapshot = fromSnapHandle.get();
+ OmSnapshot toSnapshot = toSnapHandle.get();
+ Path fromSnapshotPath =
fromSnapshot.getMetadataManager().getStore().getDbLocation().getAbsoluteFile().toPath();
+ Path toSnapshotPath =
toSnapshot.getMetadataManager().getStore().getDbLocation().getAbsoluteFile().toPath();
+ Map<Path, Pair<Path, SstFileInfo>> paths = new HashMap<>();
+ try {
+ Map<Object, SstFileInfo> fromSnapshotFiles =
getSSTFileMapForSnapshot(fromSnapshot, tablesToLookup,
+ tablePrefixInfo);
+ Map<Object, SstFileInfo> toSnapshotFiles =
getSSTFileMapForSnapshot(toSnapshot, tablesToLookup,
+ tablePrefixInfo);
+ for (Map.Entry<Object, SstFileInfo> entry :
fromSnapshotFiles.entrySet()) {
+ if (!toSnapshotFiles.containsKey(entry.getKey())) {
+ Path source = entry.getValue().getFilePath(fromSnapshotPath);
+ paths.put(source, Pair.of(createLink(source), entry.getValue()));
+ }
+ }
+ for (Map.Entry<Object, SstFileInfo> entry :
toSnapshotFiles.entrySet()) {
+ if (!fromSnapshotFiles.containsKey(entry.getKey())) {
+ Path source = entry.getValue().getFilePath(toSnapshotPath);
+ paths.put(source, Pair.of(createLink(source), entry.getValue()));
+ }
+ }
+ } catch (IOException e) {
+ // In case of exception during inode read use all files
+ LOG.error("Exception occurred while populating delta files for
snapDiff", e);
+ LOG.warn("Falling back to full file list comparison, inode-based
optimization skipped.");
+ paths.clear();
+ Set<SstFileInfo> fromSnapshotFiles =
getSSTFileSetForSnapshot(fromSnapshot, tablesToLookup, tablePrefixInfo);
+ Set<SstFileInfo> toSnapshotFiles =
getSSTFileSetForSnapshot(toSnapshot, tablesToLookup, tablePrefixInfo);
+ for (SstFileInfo sstFileInfo : fromSnapshotFiles) {
+ Path source = sstFileInfo.getFilePath(fromSnapshotPath);
+ paths.put(source, Pair.of(createLink(source), sstFileInfo));
+ }
+ for (SstFileInfo sstFileInfo : toSnapshotFiles) {
+ Path source = sstFileInfo.getFilePath(toSnapshotPath);
+ paths.put(source, Pair.of(createLink(source), sstFileInfo));
+ }
+ }
+ return Optional.of(paths);
+ }
+ }
+
+ static Map<Object, SstFileInfo> getSSTFileMapForSnapshot(OmSnapshot snapshot,
+ Set<String> tablesToLookUp, TablePrefixInfo tablePrefixInfo) throws
IOException {
+ return
filterRelevantSstFiles(RdbUtil.getSSTFilesWithInodesForComparison(((RDBStore)snapshot.getMetadataManager()
+ .getStore()).getDb().getManagedRocksDb(), tablesToLookUp),
tablesToLookUp, tablePrefixInfo);
+ }
+
+ static Set<SstFileInfo> getSSTFileSetForSnapshot(OmSnapshot snapshot,
Set<String> tablesToLookUp,
+ TablePrefixInfo tablePrefixInfo) {
+ return
filterRelevantSstFiles(RdbUtil.getSSTFilesForComparison(((RDBStore)snapshot.getMetadataManager().getStore())
+ .getDb().getManagedRocksDb(), tablesToLookUp), tablesToLookUp,
tablePrefixInfo);
Review Comment:
removed the annotation not relevant anymore
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
