Copilot commented on code in PR #9312: URL: https://github.com/apache/ozone/pull/9312#discussion_r2543632214
########## hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestRDBDifferComputer.java: ########## @@ -0,0 +1,535 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot.diff.delta; + +import static org.apache.hadoop.hdds.utils.IOUtils.getINode; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.UUID; +import java.util.function.Consumer; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.utils.db.RDBStore; +import org.apache.hadoop.hdds.utils.db.TablePrefixInfo; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshotLocalData; +import org.apache.hadoop.ozone.om.OmSnapshotLocalData.VersionMeta; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.snapshot.OmSnapshotLocalDataManager; +import org.apache.hadoop.ozone.om.snapshot.OmSnapshotLocalDataManager.ReadableOmSnapshotLocalDataProvider; +import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus; +import org.apache.ozone.rocksdb.util.SstFileInfo; +import org.apache.ozone.rocksdiff.DifferSnapshotInfo; +import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +/** + * Unit tests for RDBDifferComputer. + */ +public class TestRDBDifferComputer { + + @TempDir + private Path tempDir; + + @Mock + private OmSnapshotManager omSnapshotManager; + + @Mock + private OMMetadataManager activeMetadataManager; + + @Mock + private OmSnapshotLocalDataManager localDataManager; + + @Mock + private RDBStore rdbStore; + + @Mock + private RocksDBCheckpointDiffer differ; + + @Mock + private Consumer<SubStatus> activityReporter; + + private AutoCloseable mocks; + private Path deltaDirPath; + private RDBDifferComputer rdbDifferComputer; + + @BeforeEach + public void setUp() throws IOException { + mocks = MockitoAnnotations.openMocks(this); + deltaDirPath = tempDir.resolve("delta"); + when(omSnapshotManager.getSnapshotLocalDataManager()).thenReturn(localDataManager); + when(activeMetadataManager.getStore()).thenReturn(rdbStore); + when(rdbStore.getRocksDBCheckpointDiffer()).thenReturn(differ); + } + + @AfterEach + public void tearDown() throws Exception { + if (rdbDifferComputer != null) { + rdbDifferComputer.close(); + } + if (mocks != null) { + mocks.close(); + } + } + + /** + * Tests that the constructor creates RDBDifferComputer successfully with differ. + */ + @Test + public void testConstructorWithDiffer() throws IOException { + rdbDifferComputer = new RDBDifferComputer(omSnapshotManager, activeMetadataManager, + deltaDirPath, activityReporter); + + assertNotNull(rdbDifferComputer, "RDBDifferComputer should be created"); + assertTrue(Files.exists(deltaDirPath), "Delta directory should be created"); + verify(activeMetadataManager, times(1)).getStore(); + verify(rdbStore, times(1)).getRocksDBCheckpointDiffer(); + } + + /** + * Tests constructor when differ is null (fallback scenario). + */ + @Test + public void testConstructorWithNullDiffer() throws IOException { + when(rdbStore.getRocksDBCheckpointDiffer()).thenReturn(null); + + rdbDifferComputer = new RDBDifferComputer(omSnapshotManager, activeMetadataManager, + deltaDirPath, activityReporter); + + assertNotNull(rdbDifferComputer, "RDBDifferComputer should be created even with null differ"); + assertTrue(Files.exists(deltaDirPath), "Delta directory should be created"); + } + + /** + * Tests computeDeltaFiles with successful differ computation. + */ + @Test + public void testComputeDeltaFilesWithDiffer() throws IOException { + rdbDifferComputer = new RDBDifferComputer(omSnapshotManager, activeMetadataManager, + deltaDirPath, activityReporter); + + UUID fromSnapshotId = UUID.randomUUID(); + UUID toSnapshotId = UUID.randomUUID(); + SnapshotInfo fromSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap1", fromSnapshotId); + SnapshotInfo toSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap2", toSnapshotId); + Set<String> tablesToLookup = ImmutableSet.of("keyTable"); + TablePrefixInfo tablePrefixInfo = mock(TablePrefixInfo.class); + + // Mock snapshot local data + ReadableOmSnapshotLocalDataProvider snapProvider = mock(ReadableOmSnapshotLocalDataProvider.class); + OmSnapshotLocalData fromSnapshotLocalData = createMockSnapshotLocalData(fromSnapshotId, 1); + OmSnapshotLocalData toSnapshotLocalData = createMockSnapshotLocalData(toSnapshotId, 2); + + when(snapProvider.getPreviousSnapshotLocalData()).thenReturn(fromSnapshotLocalData); + when(snapProvider.getSnapshotLocalData()).thenReturn(toSnapshotLocalData); + when(localDataManager.getOmSnapshotLocalData(toSnapshotId, fromSnapshotId)).thenReturn(snapProvider); + + // Create mock SST files + Path sstFile1 = tempDir.resolve("sst1.sst"); + Path sstFile2 = tempDir.resolve("sst2.sst"); + Files.createFile(sstFile1); + Files.createFile(sstFile2); + + SstFileInfo sstFileInfo1 = new SstFileInfo("sst1.sst", "key1", "key2", "keyTable"); + SstFileInfo sstFileInfo2 = new SstFileInfo("sst2.sst", "key3", "key4", "keyTable"); + + Map<Path, SstFileInfo> differResult = new HashMap<>(); + differResult.put(sstFile1, sstFileInfo1); + differResult.put(sstFile2, sstFileInfo2); + + when(differ.getSSTDiffListWithFullPath(any(DifferSnapshotInfo.class), any(DifferSnapshotInfo.class), + any(Map.class), any(TablePrefixInfo.class), anySet())).thenReturn(Optional.of(differResult)); + + Optional<Map<Path, Pair<Path, SstFileInfo>>> result = + rdbDifferComputer.computeDeltaFiles(fromSnapshot, toSnapshot, tablesToLookup, tablePrefixInfo); + + assertTrue(result.isPresent(), "Result should be present"); + assertEquals(2, result.get().size(), "Should have 2 delta files"); + assertTrue(result.get().containsKey(sstFile1), "Should contain first SST file"); + assertTrue(result.get().containsKey(sstFile2), "Should contain second SST file"); + + // Verify links were created in delta directory + for (Map.Entry<Path, Pair<Path, SstFileInfo>> entry : result.get().entrySet()) { + Path actualPath = entry.getKey(); + Path link = entry.getValue().getLeft(); + assertEquals(differResult.get(actualPath), entry.getValue().getValue()); + assertTrue(link.startsWith(deltaDirPath), "Link should be in delta directory"); + assertTrue(Files.exists(link), "Link should exist"); + assertEquals(getINode(actualPath), getINode(link)); + } + + verify(snapProvider, times(1)).close(); + } + + /** + * Tests computeDeltaFiles when differ returns empty. + */ + @Test + public void testComputeDeltaFilesWithEmptyDifferResult() throws IOException { + rdbDifferComputer = new RDBDifferComputer(omSnapshotManager, activeMetadataManager, + deltaDirPath, activityReporter); + + UUID fromSnapshotId = UUID.randomUUID(); + UUID toSnapshotId = UUID.randomUUID(); + SnapshotInfo fromSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap1", fromSnapshotId); + SnapshotInfo toSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap2", toSnapshotId); + Set<String> tablesToLookup = ImmutableSet.of("keyTable"); + TablePrefixInfo tablePrefixInfo = mock(TablePrefixInfo.class); + + // Mock snapshot local data + ReadableOmSnapshotLocalDataProvider snapProvider = mock(ReadableOmSnapshotLocalDataProvider.class); + OmSnapshotLocalData fromSnapshotLocalData = createMockSnapshotLocalData(fromSnapshotId, 1); + OmSnapshotLocalData toSnapshotLocalData = createMockSnapshotLocalData(toSnapshotId, 2); + + when(snapProvider.getPreviousSnapshotLocalData()).thenReturn(fromSnapshotLocalData); + when(snapProvider.getSnapshotLocalData()).thenReturn(toSnapshotLocalData); + when(localDataManager.getOmSnapshotLocalData(toSnapshotId, fromSnapshotId)).thenReturn(snapProvider); + + when(differ.getSSTDiffListWithFullPath(any(DifferSnapshotInfo.class), any(DifferSnapshotInfo.class), + any(Map.class), any(TablePrefixInfo.class), anySet())).thenReturn(Optional.empty()); + + Optional<Map<Path, Pair<Path, SstFileInfo>>> result = + rdbDifferComputer.computeDeltaFiles(fromSnapshot, toSnapshot, tablesToLookup, tablePrefixInfo); + + assertFalse(result.isPresent(), "Result should be empty when differ returns empty"); + verify(snapProvider, times(1)).close(); + } + + /** + * Tests computeDeltaFiles when differ is null. + */ + @Test + public void testComputeDeltaFilesWithNullDiffer() throws IOException { + when(rdbStore.getRocksDBCheckpointDiffer()).thenReturn(null); + rdbDifferComputer = new RDBDifferComputer(omSnapshotManager, activeMetadataManager, + deltaDirPath, activityReporter); + + SnapshotInfo fromSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap1", UUID.randomUUID()); + SnapshotInfo toSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap2", UUID.randomUUID()); + Set<String> tablesToLookup = ImmutableSet.of("keyTable"); + TablePrefixInfo tablePrefixInfo = mock(TablePrefixInfo.class); + + Optional<Map<Path, Pair<Path, SstFileInfo>>> result = + rdbDifferComputer.computeDeltaFiles(fromSnapshot, toSnapshot, tablesToLookup, tablePrefixInfo); + + assertFalse(result.isPresent(), "Result should be empty when differ is null"); + } + + /** + * Tests computeDeltaFiles with multiple tables. + */ + @Test + public void testComputeDeltaFilesWithMultipleTables() throws IOException { + rdbDifferComputer = new RDBDifferComputer(omSnapshotManager, activeMetadataManager, + deltaDirPath, activityReporter); + + UUID fromSnapshotId = UUID.randomUUID(); + UUID toSnapshotId = UUID.randomUUID(); + SnapshotInfo fromSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap1", fromSnapshotId); + SnapshotInfo toSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap2", toSnapshotId); + Set<String> tablesToLookup = ImmutableSet.of("keyTable", "fileTable", "directoryTable"); + TablePrefixInfo tablePrefixInfo = mock(TablePrefixInfo.class); + + // Mock snapshot local data + ReadableOmSnapshotLocalDataProvider snapProvider = mock(ReadableOmSnapshotLocalDataProvider.class); + OmSnapshotLocalData fromSnapshotLocalData = createMockSnapshotLocalData(fromSnapshotId, 1); + OmSnapshotLocalData toSnapshotLocalData = createMockSnapshotLocalData(toSnapshotId, 2); + + when(snapProvider.getPreviousSnapshotLocalData()).thenReturn(fromSnapshotLocalData); + when(snapProvider.getSnapshotLocalData()).thenReturn(toSnapshotLocalData); + when(localDataManager.getOmSnapshotLocalData(toSnapshotId, fromSnapshotId)).thenReturn(snapProvider); + + // Create mock SST files for different tables + Path sstFile1 = tempDir.resolve("key1.sst"); + Path sstFile2 = tempDir.resolve("file1.sst"); + Path sstFile3 = tempDir.resolve("dir1.sst"); + Files.createFile(sstFile1); + Files.createFile(sstFile2); + Files.createFile(sstFile3); + + SstFileInfo sstFileInfo1 = new SstFileInfo("key1.sst", "key1", "key2", "keyTable"); + SstFileInfo sstFileInfo2 = new SstFileInfo("file1.sst", "file1", "file2", "fileTable"); + SstFileInfo sstFileInfo3 = new SstFileInfo("dir1.sst", "dir1", "dir2", "directoryTable"); + + Map<Path, SstFileInfo> differResult = new HashMap<>(); + differResult.put(sstFile1, sstFileInfo1); + differResult.put(sstFile2, sstFileInfo2); + differResult.put(sstFile3, sstFileInfo3); + + when(differ.getSSTDiffListWithFullPath(any(DifferSnapshotInfo.class), any(DifferSnapshotInfo.class), + any(Map.class), any(TablePrefixInfo.class), anySet())).thenReturn(Optional.of(differResult)); + + Optional<Map<Path, Pair<Path, SstFileInfo>>> result = + rdbDifferComputer.computeDeltaFiles(fromSnapshot, toSnapshot, tablesToLookup, tablePrefixInfo); + + assertTrue(result.isPresent(), "Result should be present"); + assertEquals(3, result.get().size(), "Should have 3 delta files from different tables"); + } + + /** + * Tests computeDeltaFiles with version mapping. + */ + @Test + public void testComputeDeltaFilesWithVersionMapping() throws IOException { + rdbDifferComputer = new RDBDifferComputer(omSnapshotManager, activeMetadataManager, + deltaDirPath, activityReporter); + + UUID fromSnapshotId = UUID.randomUUID(); + UUID toSnapshotId = UUID.randomUUID(); + SnapshotInfo fromSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap1", fromSnapshotId); + SnapshotInfo toSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap2", toSnapshotId); + Set<String> tablesToLookup = ImmutableSet.of("keyTable"); + TablePrefixInfo tablePrefixInfo = mock(TablePrefixInfo.class); + + // Mock snapshot local data with version mapping + ReadableOmSnapshotLocalDataProvider snapProvider = mock(ReadableOmSnapshotLocalDataProvider.class); + OmSnapshotLocalData fromSnapshotLocalData = createMockSnapshotLocalData(fromSnapshotId, 1); + OmSnapshotLocalData toSnapshotLocalData = createMockSnapshotLocalDataWithVersions(toSnapshotId, 2); + + when(snapProvider.getPreviousSnapshotLocalData()).thenReturn(fromSnapshotLocalData); + when(snapProvider.getSnapshotLocalData()).thenReturn(toSnapshotLocalData); + when(localDataManager.getOmSnapshotLocalData(toSnapshotId, fromSnapshotId)).thenReturn(snapProvider); + + Path sstFile = tempDir.resolve("sst1.sst"); + Files.createFile(sstFile); + SstFileInfo sstFileInfo = new SstFileInfo("sst1.sst", "key1", "key2", "keyTable"); + + Map<Path, SstFileInfo> differResult = new HashMap<>(); + differResult.put(sstFile, sstFileInfo); + + when(differ.getSSTDiffListWithFullPath(any(DifferSnapshotInfo.class), any(DifferSnapshotInfo.class), + any(Map.class), any(TablePrefixInfo.class), anySet())).thenReturn(Optional.of(differResult)); + + Optional<Map<Path, Pair<Path, SstFileInfo>>> result = + rdbDifferComputer.computeDeltaFiles(fromSnapshot, toSnapshot, tablesToLookup, tablePrefixInfo); + + assertTrue(result.isPresent(), "Result should be present"); + + // Verify that version map was passed to differ + ArgumentCaptor<Map<Integer, Integer>> versionMapCaptor = ArgumentCaptor.forClass(Map.class); + verify(differ).getSSTDiffListWithFullPath(any(DifferSnapshotInfo.class), any(DifferSnapshotInfo.class), + versionMapCaptor.capture(), any(TablePrefixInfo.class), anySet()); + + Map<Integer, Integer> capturedVersionMap = versionMapCaptor.getValue(); + assertNotNull(capturedVersionMap, "Version map should not be null"); + assertEquals(ImmutableMap.of(0, 0, 1, 0, 2, 1), capturedVersionMap); + } + + /** + * Tests that getDSIFromSI throws exception when no versions found. + */ + @Test + public void testGetDSIFromSIWithNoVersions() throws IOException { + rdbDifferComputer = new RDBDifferComputer(omSnapshotManager, activeMetadataManager, + deltaDirPath, activityReporter); + + UUID snapshotId = UUID.randomUUID(); + SnapshotInfo fromSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap1", snapshotId); + SnapshotInfo toSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap2", UUID.randomUUID()); + Set<String> tablesToLookup = ImmutableSet.of("keyTable"); + TablePrefixInfo tablePrefixInfo = mock(TablePrefixInfo.class); + + // Mock snapshot local data with empty versions + ReadableOmSnapshotLocalDataProvider snapProvider = mock(ReadableOmSnapshotLocalDataProvider.class); + OmSnapshotLocalData fromSnapshotLocalData = mock(OmSnapshotLocalData.class); + OmSnapshotLocalData toSnapshotLocalData = createMockSnapshotLocalData(UUID.randomUUID(), 1); + + when(fromSnapshotLocalData.getSnapshotId()).thenReturn(snapshotId); + when(fromSnapshotLocalData.getVersionSstFileInfos()).thenReturn(Collections.emptyMap()); + + when(snapProvider.getPreviousSnapshotLocalData()).thenReturn(fromSnapshotLocalData); + when(snapProvider.getSnapshotLocalData()).thenReturn(toSnapshotLocalData); + when(localDataManager.getOmSnapshotLocalData(any(UUID.class), any(UUID.class))).thenReturn(snapProvider); + + assertThrows(IOException.class, () -> + rdbDifferComputer.computeDeltaFiles(fromSnapshot, toSnapshot, tablesToLookup, tablePrefixInfo), + "Should throw IOException when no versions found"); + } + + /** + * Tests that close properly cleans up resources. + */ + @Test + public void testClose() throws IOException { + rdbDifferComputer = new RDBDifferComputer(omSnapshotManager, activeMetadataManager, + deltaDirPath, activityReporter); + + assertTrue(Files.exists(deltaDirPath), "Delta directory should exist"); + + rdbDifferComputer.close(); + + assertFalse(Files.exists(deltaDirPath), "Delta directory should be cleaned up after close"); + } + + /** + * Tests computeDeltaFiles with IOException from differ. + */ + @Test + public void testComputeDeltaFilesWithIOException() throws IOException { + rdbDifferComputer = new RDBDifferComputer(omSnapshotManager, activeMetadataManager, + deltaDirPath, activityReporter); + + UUID fromSnapshotId = UUID.randomUUID(); + UUID toSnapshotId = UUID.randomUUID(); + SnapshotInfo fromSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap1", fromSnapshotId); + SnapshotInfo toSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap2", toSnapshotId); + Set<String> tablesToLookup = ImmutableSet.of("keyTable"); + TablePrefixInfo tablePrefixInfo = mock(TablePrefixInfo.class); + + // Mock snapshot local data + ReadableOmSnapshotLocalDataProvider snapProvider = mock(ReadableOmSnapshotLocalDataProvider.class); + OmSnapshotLocalData fromSnapshotLocalData = createMockSnapshotLocalData(fromSnapshotId, 1); + OmSnapshotLocalData toSnapshotLocalData = createMockSnapshotLocalData(toSnapshotId, 2); + + when(snapProvider.getPreviousSnapshotLocalData()).thenReturn(fromSnapshotLocalData); + when(snapProvider.getSnapshotLocalData()).thenReturn(toSnapshotLocalData); + when(localDataManager.getOmSnapshotLocalData(toSnapshotId, fromSnapshotId)).thenReturn(snapProvider); + + when(differ.getSSTDiffListWithFullPath(any(DifferSnapshotInfo.class), any(DifferSnapshotInfo.class), + any(Map.class), any(TablePrefixInfo.class), anySet())) + .thenThrow(new IOException("Test exception")); + + assertThrows(IOException.class, () -> + rdbDifferComputer.computeDeltaFiles(fromSnapshot, toSnapshot, tablesToLookup, tablePrefixInfo), + "Should propagate IOException from differ"); + + verify(snapProvider, times(1)).close(); + } + + /** + * Tests that differ operations are synchronized. + */ + @Test + public void testDifferSynchronization() throws IOException { + rdbDifferComputer = new RDBDifferComputer(omSnapshotManager, activeMetadataManager, + deltaDirPath, activityReporter); + + UUID fromSnapshotId = UUID.randomUUID(); + UUID toSnapshotId = UUID.randomUUID(); + SnapshotInfo fromSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap1", fromSnapshotId); + SnapshotInfo toSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap2", toSnapshotId); + Set<String> tablesToLookup = ImmutableSet.of("keyTable"); + TablePrefixInfo tablePrefixInfo = mock(TablePrefixInfo.class); + + // Mock snapshot local data + ReadableOmSnapshotLocalDataProvider snapProvider = mock(ReadableOmSnapshotLocalDataProvider.class); + OmSnapshotLocalData fromSnapshotLocalData = createMockSnapshotLocalData(fromSnapshotId, 1); + OmSnapshotLocalData toSnapshotLocalData = createMockSnapshotLocalData(toSnapshotId, 2); + + when(snapProvider.getPreviousSnapshotLocalData()).thenReturn(fromSnapshotLocalData); + when(snapProvider.getSnapshotLocalData()).thenReturn(toSnapshotLocalData); + when(localDataManager.getOmSnapshotLocalData(toSnapshotId, fromSnapshotId)).thenReturn(snapProvider); + + when(differ.getSSTDiffListWithFullPath(any(DifferSnapshotInfo.class), any(DifferSnapshotInfo.class), + any(Map.class), any(TablePrefixInfo.class), anySet())).thenReturn(Optional.empty()); + + // Multiple calls should work correctly (synchronized access to differ) + for (int i = 0; i < 3; i++) { + Optional<Map<Path, Pair<Path, SstFileInfo>>> result = + rdbDifferComputer.computeDeltaFiles(fromSnapshot, toSnapshot, tablesToLookup, tablePrefixInfo); + assertFalse(result.isPresent(), "Result should be empty"); + } + + verify(differ, times(3)).getSSTDiffListWithFullPath(any(DifferSnapshotInfo.class), + any(DifferSnapshotInfo.class), any(Map.class), any(TablePrefixInfo.class), anySet()); + } + + // Helper methods + + private SnapshotInfo createMockSnapshotInfo(String volumeName, String bucketName, + String snapshotName, UUID snapshotId) { + SnapshotInfo.Builder builder = SnapshotInfo.newBuilder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setName(snapshotName) + .setSnapshotId(snapshotId) + .setDbTxSequenceNumber(100L); + return builder.build(); + } + + private OmSnapshotLocalData createMockSnapshotLocalData(UUID snapshotId, int version) { + OmSnapshotLocalData localData = mock(OmSnapshotLocalData.class); + when(localData.getSnapshotId()).thenReturn(snapshotId); + + // Create version SST file info + List<SstFileInfo> sstFiles = new ArrayList<>(); + sstFiles.add(new SstFileInfo("file1.sst", "key1", "key2", "keyTable")); + + VersionMeta versionMeta = new VersionMeta(0, sstFiles); + Map<Integer, VersionMeta> versionMap = new TreeMap<>(); + versionMap.put(version, versionMeta); + + when(localData.getVersionSstFileInfos()).thenReturn(versionMap); + when(localData.getVersion()).thenReturn(version); + + return localData; + } + + private OmSnapshotLocalData createMockSnapshotLocalDataWithVersions(UUID snapshotId, int version) { + OmSnapshotLocalData localData = mock(OmSnapshotLocalData.class); + when(localData.getSnapshotId()).thenReturn(snapshotId); + + // Create multiple versions + Map<Integer, VersionMeta> versionMap = new TreeMap<>(); + for (int i = 0; i <= version; i++) { + List<SstFileInfo> sstFiles = new ArrayList<>(); + sstFiles.add(new SstFileInfo("file" + i + ".sst", "key" + i, "key" + (i + 1), "keyTable")); + VersionMeta versionMeta = new VersionMeta(i > 0 ? i - 1 : 0, sstFiles); + versionMap.put(i, versionMeta); + } + + when(localData.getVersionSstFileInfos()).thenReturn(versionMap); + when(localData.getVersion()).thenReturn(version); + + return localData; + } +} + + + + + Review Comment: [nitpick] Extra blank lines at the end of the file. Remove lines 532-535 to leave only a single blank line at the end of the file, following common Java style conventions. ```suggestion ``` ########## hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/RDBDifferComputer.java: ########## @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot.diff.delta; + +import static java.util.stream.Collectors.toMap; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.UUID; +import java.util.function.Consumer; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.utils.db.TablePrefixInfo; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshotLocalData; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.snapshot.OmSnapshotLocalDataManager; +import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus; +import org.apache.ozone.rocksdb.util.SstFileInfo; +import org.apache.ozone.rocksdiff.DifferSnapshotInfo; +import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer; + +/** + * Computes RocksDB SST file differences between two snapshots and materializes + * differing SST files as hard links in the configured delta directory. + * + * <p>This class uses {@link RocksDBCheckpointDiffer} to obtain the list of SST + * files that differ between a \"from\" and a \"to\" snapshot. It opens local + * snapshot metadata via {@link #getLocalDataProvider}, and delegates the + * comparison to the differ to compute the delta files.</p> + * + * <p>Each source SST file returned by the differ is linked into the delta + * directory using {@link FileLinkDeltaFileComputer#createLink(Path)}, and the + * returned value from {@link #computeDeltaFiles} is a list of those link + * paths. The implementation synchronizes on the internal {@code differ} + * instance because the differ is not assumed to be thread-safe.</p> + */ +class RDBDifferComputer extends FileLinkDeltaFileComputer { + + private final RocksDBCheckpointDiffer differ; + + RDBDifferComputer(OmSnapshotManager omSnapshotManager, OMMetadataManager activeMetadataManager, + Path deltaDirPath, Consumer<SubStatus> activityReporter) throws IOException { + super(omSnapshotManager, activeMetadataManager, deltaDirPath, activityReporter); + this.differ = activeMetadataManager.getStore().getRocksDBCheckpointDiffer(); Review Comment: Access of [element](1) annotated with VisibleForTesting found in production code. ```suggestion Path deltaDirPath, Consumer<SubStatus> activityReporter, RocksDBCheckpointDiffer differ) throws IOException { super(omSnapshotManager, activeMetadataManager, deltaDirPath, activityReporter); this.differ = differ; ``` ########## hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/RDBDifferComputer.java: ########## @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot.diff.delta; + +import static java.util.stream.Collectors.toMap; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.UUID; +import java.util.function.Consumer; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.utils.db.TablePrefixInfo; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshotLocalData; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.snapshot.OmSnapshotLocalDataManager; +import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus; +import org.apache.ozone.rocksdb.util.SstFileInfo; +import org.apache.ozone.rocksdiff.DifferSnapshotInfo; +import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer; + +/** + * Computes RocksDB SST file differences between two snapshots and materializes + * differing SST files as hard links in the configured delta directory. + * + * <p>This class uses {@link RocksDBCheckpointDiffer} to obtain the list of SST + * files that differ between a \"from\" and a \"to\" snapshot. It opens local Review Comment: The escaped quote in the javadoc comment should be a regular quote. Change `\"from\"` and `\"to\"` to just `"from"` and `"to"` as javadoc doesn't require escaping quotes. ```suggestion * files that differ between a "from" and a "to" snapshot. It opens local ``` ########## hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java: ########## @@ -772,15 +771,13 @@ private String getSSTFullPath(String sstFilenameWithoutExtension, Path... dbPath * @param dest destination snapshot * @param versionMap version map containing the connection between source snapshot version and dest snapshot version. * @param tablesToLookup tablesToLookup set of table (column family) names used to restrict which SST files to return. - * @param sstFilesDirForSnapDiffJob dir to create hardlinks for SST files - * for snapDiff job. * @return A list of SST files without extension. * e.g. ["/path/to/sstBackupDir/000050.sst", * "/path/to/sstBackupDir/000060.sst"] Review Comment: The return description in the javadoc is outdated. The method now returns `Optional<Map<Path, SstFileInfo>>` but the documentation still says "A list of SST files without extension." Update this to accurately describe the return type as a Map of SST file paths to their corresponding SstFileInfo metadata. ########## hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/FileLinkDeltaFileComputer.java: ########## @@ -49,7 +49,8 @@ /** * The {@code FileLinkDeltaFileComputer} is an abstract class that provides a * base implementation for the {@code DeltaFileComputer} interface. It is - * responsible for computing delta files by creating hard links to the + * responsible for computing delta files (a list of files if read completely would be able to completely + * compute all the key changes between two snapshots). Hard links to the * relevant source files in a specified delta directory, enabling a compact * representation of changes between snapshots. Review Comment: Documentation is grammatically incomplete. Lines 52-55 form a sentence fragment. Consider rewording to: "responsible for computing delta files (a list of files that, if read completely, would enable computation of all key changes between two snapshots) and creating hard links to the relevant source files in a specified delta directory, enabling a compact representation of changes between snapshots." ```suggestion * responsible for computing delta files (a list of files that, if read completely, would enable computation of all key changes between two snapshots) and creating hard links to the relevant source files in a specified delta directory, enabling a compact representation of changes between snapshots. ``` ########## hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/CompositeDeltaDiffComputer.java: ########## @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot.diff.delta; + +import static org.apache.hadoop.ozone.om.snapshot.diff.delta.FullDiffComputer.getSSTFileSetForSnapshot; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.utils.db.TablePrefixInfo; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; +import org.apache.ozone.rocksdb.util.SstFileInfo; +import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * CompositeDeltaDiffComputer is responsible for computing the delta file + * differences between two snapshots, utilizing different strategies such + * as partial differ computation and full differ computation. + * + * It serves as an orchestrator to decide whether to perform a full diff + * or a more efficient partial diff, and handles fallback mechanisms if + * the chosen method fails. + * + * The class leverages two main difference computation strategies: + * - {@code RDBDifferComputer} for partial diff computation + * - {@code FullDiffComputer} for exhaustive diff + * + * This class also includes support for handling non-native diff scenarios + * through additional processing of input files from the "from" snapshot + * when native RocksDB tools are not used. + * + * Inherits from {@code FileLinkDeltaFileComputer} and implements the + * functionality for computing delta files and resource management. + */ +public class CompositeDeltaDiffComputer extends FileLinkDeltaFileComputer { + + private static final Logger LOG = LoggerFactory.getLogger(CompositeDeltaDiffComputer.class); + + private final RDBDifferComputer differComputer; + private final FullDiffComputer fullDiffComputer; + private final boolean nonNativeDiff; + + public CompositeDeltaDiffComputer(OmSnapshotManager snapshotManager, + OMMetadataManager activeMetadataManager, Path deltaDirPath, + Consumer<SnapshotDiffResponse.SubStatus> activityReporter, boolean fullDiff, + boolean nonNativeDiff) throws IOException { + super(snapshotManager, activeMetadataManager, deltaDirPath, activityReporter); + differComputer = fullDiff ? null : new RDBDifferComputer(snapshotManager, activeMetadataManager, + deltaDirPath.resolve("rdbDiffer"), activityReporter); + fullDiffComputer = new FullDiffComputer(snapshotManager, activeMetadataManager, + deltaDirPath.resolve("fullDiff"), activityReporter); + this.nonNativeDiff = nonNativeDiff; + } + + @Override + Optional<Map<Path, Pair<Path, SstFileInfo>>> computeDeltaFiles(SnapshotInfo fromSnapshotInfo, + SnapshotInfo toSnapshotInfo, Set<String> tablesToLookup, TablePrefixInfo tablePrefixInfo) throws IOException { + Map<Path, Pair<Path, SstFileInfo>> deltaFiles = null; + try { + if (differComputer != null) { + updateActivity(SnapshotDiffResponse.SubStatus.SST_FILE_DELTA_DAG_WALK); + deltaFiles = differComputer.computeDeltaFiles(fromSnapshotInfo, toSnapshotInfo, tablesToLookup, + tablePrefixInfo).orElse(null); + } + } catch (Exception e) { + LOG.warn("Falling back to full diff.", e); + } + if (deltaFiles == null) { + updateActivity(SnapshotDiffResponse.SubStatus.SST_FILE_DELTA_FULL_DIFF); + deltaFiles = fullDiffComputer.computeDeltaFiles(fromSnapshotInfo, toSnapshotInfo, tablesToLookup, + tablePrefixInfo).orElse(null); + if (deltaFiles == null) { + // FileLinkDeltaFileComputer would throw an exception in this case. + return Optional.empty(); + } + } + // Workaround to handle deletes if native rocksDb tool for reading + // tombstone is not loaded. + // When performing non native diff, input files of from snapshot needs to be added. Review Comment: Grammar error in comment. Change "input files of from snapshot needs to be added" to "input files from the from snapshot need to be added" (or "from the source snapshot need to be added" for clarity). ```suggestion // When performing non native diff, input files from the 'from' snapshot need to be added. ``` ########## hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestCompositeDeltaDiffComputer.java: ########## @@ -0,0 +1,726 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot.diff.delta; + +import static org.apache.hadoop.hdds.utils.IOUtils.getINode; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.utils.db.RDBStore; +import org.apache.hadoop.hdds.utils.db.TablePrefixInfo; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.snapshot.OmSnapshotLocalDataManager; +import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus; +import org.apache.ozone.rocksdb.util.SstFileInfo; +import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer; +import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.MockitoAnnotations; + +/** + * Unit tests for CompositeDeltaDiffComputer using Mockito.mockConstruction() + * to properly isolate and test fallback logic. + */ +public class TestCompositeDeltaDiffComputer { + + @TempDir + private Path tempDir; + + @Mock + private OmSnapshotManager omSnapshotManager; + + @Mock + private OMMetadataManager activeMetadataManager; + + @Mock + private OmSnapshotLocalDataManager localDataManager; + + @Mock + private RDBStore rdbStore; + + @Mock + private RocksDBCheckpointDiffer differ; + + @Mock + private Consumer<SubStatus> activityReporter; + + private AutoCloseable mocks; + private Path deltaDirPath; + + @BeforeEach + public void setUp() throws IOException { + mocks = MockitoAnnotations.openMocks(this); + deltaDirPath = tempDir.resolve("delta"); + when(omSnapshotManager.getSnapshotLocalDataManager()).thenReturn(localDataManager); + when(activeMetadataManager.getStore()).thenReturn(rdbStore); + when(rdbStore.getRocksDBCheckpointDiffer()).thenReturn(differ); + } + + @AfterEach + public void tearDown() throws Exception { + if (mocks != null) { + mocks.close(); + } + } + + /** + * Tests that RDBDifferComputer is created when fullDiff=false using mockConstruction. + */ + @Test + public void testRDBDifferComputerCreatedWhenNotFullDiff() throws IOException { + try (MockedConstruction<RDBDifferComputer> rdbDifferMock = mockConstruction(RDBDifferComputer.class); + MockedConstruction<FullDiffComputer> fullDiffMock = mockConstruction(FullDiffComputer.class)) { + + CompositeDeltaDiffComputer composite = new CompositeDeltaDiffComputer( + omSnapshotManager, activeMetadataManager, deltaDirPath, activityReporter, false, false); + + // Verify RDBDifferComputer was constructed (fullDiff=false) + assertEquals(1, rdbDifferMock.constructed().size(), "RDBDifferComputer should be constructed"); + assertEquals(1, fullDiffMock.constructed().size(), "FullDiffComputer should always be constructed"); + + composite.close(); + } + } + + /** + * Tests that RDBDifferComputer is NOT created when fullDiff=true using mockConstruction. + */ + @Test + public void testRDBDifferComputerNotCreatedWhenFullDiff() throws IOException { + try (MockedConstruction<RDBDifferComputer> rdbDifferMock = mockConstruction(RDBDifferComputer.class); + MockedConstruction<FullDiffComputer> fullDiffMock = mockConstruction(FullDiffComputer.class)) { + + CompositeDeltaDiffComputer composite = new CompositeDeltaDiffComputer( + omSnapshotManager, activeMetadataManager, deltaDirPath, activityReporter, true, false); + + // Verify RDBDifferComputer was NOT constructed (fullDiff=true) + assertEquals(0, rdbDifferMock.constructed().size(), "RDBDifferComputer should NOT " + + "be constructed when fullDiff=true"); + assertEquals(1, fullDiffMock.constructed().size(), "FullDiffComputer should always be constructed"); + + composite.close(); + } + } + + /** + * Tests successful RDBDifferComputer computation without fallback. + */ + @Test + public void testSuccessfulRDBDifferComputationWithoutFallback() throws IOException { + UUID fromSnapshotId = UUID.randomUUID(); + UUID toSnapshotId = UUID.randomUUID(); + SnapshotInfo fromSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap1", fromSnapshotId); + SnapshotInfo toSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap2", toSnapshotId); + Set<String> tablesToLookup = ImmutableSet.of("keyTable"); + TablePrefixInfo tablePrefixInfo = new TablePrefixInfo(ImmutableMap.of("keyTable", "a")); + + // Create expected results from RDBDiffer + Path sstFile1 = tempDir.resolve("rdb1.sst"); + Path sstFile2 = tempDir.resolve("rdb2.sst"); + Files.createFile(sstFile1); + Files.createFile(sstFile2); + SstFileInfo sstInfo1 = new SstFileInfo("rdb1.sst", "key1", "key2", "keyTable"); + SstFileInfo sstInfo2 = new SstFileInfo("rdb2.sst", "key3", "key4", "keyTable"); + Map<Path, Pair<Path, SstFileInfo>> rdbDifferResult = new HashMap<>(); + rdbDifferResult.put(sstFile1, Pair.of(sstFile1, sstInfo1)); + rdbDifferResult.put(sstFile2, Pair.of(sstFile2, sstInfo2)); + + try (MockedConstruction<RDBDifferComputer> rdbDifferMock = mockConstruction(RDBDifferComputer.class, + (mock, context) -> { + // Make RDBDifferComputer return results successfully + when(mock.computeDeltaFiles(any(), any(), anySet(), any())) + .thenReturn(Optional.of(rdbDifferResult)); + }); + MockedConstruction<FullDiffComputer> fullDiffMock = mockConstruction(FullDiffComputer.class)) { + + CompositeDeltaDiffComputer composite = new CompositeDeltaDiffComputer( + omSnapshotManager, activeMetadataManager, deltaDirPath, activityReporter, false, false); + + Optional<Map<Path, Pair<Path, SstFileInfo>>> result = + composite.computeDeltaFiles(fromSnapshot, toSnapshot, tablesToLookup, tablePrefixInfo); + + // Verify RDBDiffer results are returned + assertTrue(result.isPresent(), "Result should be present from RDBDiffer"); + assertEquals(2, result.get().size(), "Should have 2 files from RDBDiffer"); + assertEquals(rdbDifferResult, result.get(), "Should return RDBDifferComputer result"); + + // Verify RDBDifferComputer was called but NOT FullDiffComputer + RDBDifferComputer rdbDifferInstance = rdbDifferMock.constructed().get(0); + verify(rdbDifferInstance, times(1)).computeDeltaFiles(any(), any(), anySet(), any()); + + // Verify FullDiffComputer was NEVER called (no fallback needed) + FullDiffComputer fullDiffInstance = fullDiffMock.constructed().get(0); + verify(fullDiffInstance, times(0)).computeDeltaFiles(any(), any(), anySet(), any()); + + // Verify only DAG_WALK status was reported (no FULL_DIFF) + ArgumentCaptor<SubStatus> statusCaptor = ArgumentCaptor.forClass(SubStatus.class); + verify(activityReporter, times(1)).accept(statusCaptor.capture()); + assertEquals(SubStatus.SST_FILE_DELTA_DAG_WALK, statusCaptor.getValue(), + "Only DAG_WALK should be reported when RDBDiffer succeeds"); + + composite.close(); + } + } + + /** + * Tests successful RDBDifferComputer with single file. + */ + @Test + public void testSuccessfulRDBDifferWithSingleFile() throws IOException { + UUID fromSnapshotId = UUID.randomUUID(); + UUID toSnapshotId = UUID.randomUUID(); + SnapshotInfo fromSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap1", fromSnapshotId); + SnapshotInfo toSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap2", toSnapshotId); + Set<String> tablesToLookup = ImmutableSet.of("keyTable"); + TablePrefixInfo tablePrefixInfo = new TablePrefixInfo(ImmutableMap.of("keyTable", "a")); + + Path sstFile = tempDir.resolve("single.sst"); + Files.createFile(sstFile); + SstFileInfo sstInfo = new SstFileInfo("single.sst", "key1", "key5", "keyTable"); + Map<Path, Pair<Path, SstFileInfo>> rdbDifferResult = new HashMap<>(); + rdbDifferResult.put(sstFile, Pair.of(sstFile, sstInfo)); + + try (MockedConstruction<RDBDifferComputer> rdbDifferMock = mockConstruction(RDBDifferComputer.class, + (mock, context) -> { + when(mock.computeDeltaFiles(any(), any(), anySet(), any())) + .thenReturn(Optional.of(rdbDifferResult)); + }); + MockedConstruction<FullDiffComputer> fullDiffMock = mockConstruction(FullDiffComputer.class)) { + + CompositeDeltaDiffComputer composite = new CompositeDeltaDiffComputer( + omSnapshotManager, activeMetadataManager, deltaDirPath, activityReporter, false, false); + + Optional<Map<Path, Pair<Path, SstFileInfo>>> result = + composite.computeDeltaFiles(fromSnapshot, toSnapshot, tablesToLookup, tablePrefixInfo); + + assertTrue(result.isPresent(), "Result should be present"); + assertEquals(1, result.get().size(), "Should have 1 file"); + + // Verify no fallback to FullDiff + FullDiffComputer fullDiffInstance = fullDiffMock.constructed().get(0); + verify(fullDiffInstance, times(0)).computeDeltaFiles(any(), any(), anySet(), any()); + + composite.close(); + } + } + + /** + * Tests successful RDBDifferComputer with multiple tables. + */ + @Test + public void testSuccessfulRDBDifferWithMultipleTables() throws IOException { + UUID fromSnapshotId = UUID.randomUUID(); + UUID toSnapshotId = UUID.randomUUID(); + SnapshotInfo fromSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap1", fromSnapshotId); + SnapshotInfo toSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap2", toSnapshotId); + Set<String> tablesToLookup = ImmutableSet.of("keyTable", "fileTable", "directoryTable"); + TablePrefixInfo tablePrefixInfo = new TablePrefixInfo(ImmutableMap.of( + "keyTable", "a", "fileTable", "b", "directoryTable", "c")); + + // Create files for different tables + Path keyFile = tempDir.resolve("key1.sst"); + Path fileFile = tempDir.resolve("file1.sst"); + Path dirFile = tempDir.resolve("dir1.sst"); + Files.createFile(keyFile); + Files.createFile(fileFile); + Files.createFile(dirFile); + + SstFileInfo keyInfo = new SstFileInfo("key1.sst", "key1", "key2", "keyTable"); + SstFileInfo fileInfo = new SstFileInfo("file1.sst", "file1", "file2", "fileTable"); + SstFileInfo dirInfo = new SstFileInfo("dir1.sst", "dir1", "dir2", "directoryTable"); + + Map<Path, Pair<Path, SstFileInfo>> rdbDifferResult = new HashMap<>(); + rdbDifferResult.put(keyFile, Pair.of(keyFile, keyInfo)); + rdbDifferResult.put(fileFile, Pair.of(fileFile, fileInfo)); + rdbDifferResult.put(dirFile, Pair.of(dirFile, dirInfo)); + + try (MockedConstruction<RDBDifferComputer> rdbDifferMock = mockConstruction(RDBDifferComputer.class, + (mock, context) -> { + when(mock.computeDeltaFiles(any(), any(), anySet(), any())) + .thenReturn(Optional.of(rdbDifferResult)); + }); + MockedConstruction<FullDiffComputer> fullDiffMock = mockConstruction(FullDiffComputer.class)) { + + CompositeDeltaDiffComputer composite = new CompositeDeltaDiffComputer( + omSnapshotManager, activeMetadataManager, deltaDirPath, activityReporter, false, false); + + Optional<Map<Path, Pair<Path, SstFileInfo>>> result = + composite.computeDeltaFiles(fromSnapshot, toSnapshot, tablesToLookup, tablePrefixInfo); + + assertTrue(result.isPresent(), "Result should be present"); + assertEquals(3, result.get().size(), "Should have 3 files from different tables"); + + // Verify RDBDiffer handled all tables without fallback + RDBDifferComputer rdbDifferInstance = rdbDifferMock.constructed().get(0); + verify(rdbDifferInstance, times(1)).computeDeltaFiles(any(), any(), anySet(), any()); + + FullDiffComputer fullDiffInstance = fullDiffMock.constructed().get(0); + verify(fullDiffInstance, times(0)).computeDeltaFiles(any(), any(), anySet(), any()); + + composite.close(); + } + } + + /** + * Tests successful RDBDifferComputer returning empty map (no changes). + */ + @Test + public void testSuccessfulRDBDifferWithNoChanges() throws IOException { + UUID fromSnapshotId = UUID.randomUUID(); + UUID toSnapshotId = UUID.randomUUID(); + SnapshotInfo fromSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap1", fromSnapshotId); + SnapshotInfo toSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap2", toSnapshotId); + Set<String> tablesToLookup = ImmutableSet.of("keyTable"); + TablePrefixInfo tablePrefixInfo = new TablePrefixInfo(ImmutableMap.of("keyTable", "a")); + + // RDBDiffer returns empty map (no differences, but successful computation) + Map<Path, Pair<Path, SstFileInfo>> emptyResult = new HashMap<>(); + + try (MockedConstruction<RDBDifferComputer> rdbDifferMock = mockConstruction(RDBDifferComputer.class, + (mock, context) -> { + when(mock.computeDeltaFiles(any(), any(), anySet(), any())) + .thenReturn(Optional.of(emptyResult)); + }); + MockedConstruction<FullDiffComputer> fullDiffMock = mockConstruction(FullDiffComputer.class)) { + + CompositeDeltaDiffComputer composite = new CompositeDeltaDiffComputer( + omSnapshotManager, activeMetadataManager, deltaDirPath, activityReporter, false, false); + + Optional<Map<Path, Pair<Path, SstFileInfo>>> result = + composite.computeDeltaFiles(fromSnapshot, toSnapshot, tablesToLookup, tablePrefixInfo); + + // Empty result is still a valid success case - no fallback needed + assertTrue(result.isPresent(), "Result should be present even if empty"); + assertEquals(0, result.get().size(), "Should have 0 files (no changes)"); + + // Verify no fallback occurred + FullDiffComputer fullDiffInstance = fullDiffMock.constructed().get(0); + verify(fullDiffInstance, times(0)).computeDeltaFiles(any(), any(), anySet(), any()); + + // Only DAG_WALK status should be reported + ArgumentCaptor<SubStatus> statusCaptor = ArgumentCaptor.forClass(SubStatus.class); + verify(activityReporter, times(1)).accept(statusCaptor.capture()); + assertEquals(SubStatus.SST_FILE_DELTA_DAG_WALK, statusCaptor.getValue()); + + composite.close(); + } + } + + /** + * Tests fallback from RDBDifferComputer to FullDiffComputer using mockConstruction. + */ + @Test + public void testFallbackFromRDBDifferToFullDiff() throws IOException { + UUID fromSnapshotId = UUID.randomUUID(); + UUID toSnapshotId = UUID.randomUUID(); + SnapshotInfo fromSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap1", fromSnapshotId); + SnapshotInfo toSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap2", toSnapshotId); + Set<String> tablesToLookup = ImmutableSet.of("keyTable"); + TablePrefixInfo tablePrefixInfo = new TablePrefixInfo(ImmutableMap.of("keyTable", "a")); + + // Create expected results + Path sstFile = tempDir.resolve("test.sst"); + Files.createFile(sstFile); + SstFileInfo sstInfo = new SstFileInfo("test.sst", "key1", "key2", "keyTable"); + Map<Path, Pair<Path, SstFileInfo>> fullDiffResult = new HashMap<>(); + fullDiffResult.put(sstFile, Pair.of(sstFile, sstInfo)); + + try (MockedConstruction<RDBDifferComputer> rdbDifferMock = mockConstruction(RDBDifferComputer.class, + (mock, context) -> { + // Make RDBDifferComputer return empty to trigger fallback + when(mock.computeDeltaFiles(any(), any(), anySet(), any())) + .thenReturn(Optional.empty()); + }); + MockedConstruction<FullDiffComputer> fullDiffMock = mockConstruction(FullDiffComputer.class, + (mock, context) -> { + // Make FullDiffComputer return results + when(mock.computeDeltaFiles(any(), any(), anySet(), any())) + .thenReturn(Optional.of(fullDiffResult)); + })) { + + CompositeDeltaDiffComputer composite = new CompositeDeltaDiffComputer( + omSnapshotManager, activeMetadataManager, deltaDirPath, activityReporter, false, false); + + Optional<Map<Path, Pair<Path, SstFileInfo>>> result = + composite.computeDeltaFiles(fromSnapshot, toSnapshot, tablesToLookup, tablePrefixInfo); + + // Verify fallback occurred + assertTrue(result.isPresent(), "Result should be present from fallback"); + assertEquals(fullDiffResult, result.get(), "Should return FullDiffComputer result"); + + // Verify both computers were called + RDBDifferComputer rdbDifferInstance = rdbDifferMock.constructed().get(0); + FullDiffComputer fullDiffInstance = fullDiffMock.constructed().get(0); + + verify(rdbDifferInstance, times(1)).computeDeltaFiles(any(), any(), anySet(), any()); + verify(fullDiffInstance, times(1)).computeDeltaFiles(any(), any(), anySet(), any()); + + // Verify activity statuses were reported + ArgumentCaptor<SubStatus> statusCaptor = ArgumentCaptor.forClass(SubStatus.class); + verify(activityReporter, times(2)).accept(statusCaptor.capture()); + List<SubStatus> statuses = statusCaptor.getAllValues(); + assertEquals(SubStatus.SST_FILE_DELTA_DAG_WALK, statuses.get(0)); + assertEquals(SubStatus.SST_FILE_DELTA_FULL_DIFF, statuses.get(1)); + + composite.close(); + } + } + + /** + * Tests fallback on exception using mockConstruction. + */ + @Test + public void testFallbackOnException() throws IOException { + UUID fromSnapshotId = UUID.randomUUID(); + UUID toSnapshotId = UUID.randomUUID(); + SnapshotInfo fromSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap1", fromSnapshotId); + SnapshotInfo toSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap2", toSnapshotId); + Set<String> tablesToLookup = ImmutableSet.of("keyTable"); + TablePrefixInfo tablePrefixInfo = new TablePrefixInfo(ImmutableMap.of("keyTable", "a")); + + Path sstFile = tempDir.resolve("test2.sst"); + Files.createFile(sstFile); + SstFileInfo sstInfo = new SstFileInfo("test2.sst", "key3", "key4", "keyTable"); + Map<Path, Pair<Path, SstFileInfo>> fullDiffResult = new HashMap<>(); + fullDiffResult.put(sstFile, Pair.of(sstFile, sstInfo)); + + try (MockedConstruction<RDBDifferComputer> rdbDifferMock = mockConstruction(RDBDifferComputer.class, + (mock, context) -> { + // Make RDBDifferComputer throw exception to trigger fallback + when(mock.computeDeltaFiles(any(), any(), anySet(), any())) + .thenThrow(new RuntimeException("Test exception")); + }); + MockedConstruction<FullDiffComputer> fullDiffMock = mockConstruction(FullDiffComputer.class, + (mock, context) -> { + // Make FullDiffComputer return results + when(mock.computeDeltaFiles(any(), any(), anySet(), any())) + .thenReturn(Optional.of(fullDiffResult)); + })) { + + CompositeDeltaDiffComputer composite = new CompositeDeltaDiffComputer( + omSnapshotManager, activeMetadataManager, deltaDirPath, activityReporter, false, false); + + Optional<Map<Path, Pair<Path, SstFileInfo>>> result = + composite.computeDeltaFiles(fromSnapshot, toSnapshot, tablesToLookup, tablePrefixInfo); + + // Verify fallback occurred + assertTrue(result.isPresent(), "Result should be present from fallback after exception"); + + // Verify activity statuses + ArgumentCaptor<SubStatus> statusCaptor = ArgumentCaptor.forClass(SubStatus.class); + verify(activityReporter, times(2)).accept(statusCaptor.capture()); + List<SubStatus> statuses = statusCaptor.getAllValues(); + assertEquals(SubStatus.SST_FILE_DELTA_DAG_WALK, statuses.get(0)); + assertEquals(SubStatus.SST_FILE_DELTA_FULL_DIFF, statuses.get(1)); + + composite.close(); + } + } + + /** + * Tests that FullDiffComputer is used directly when fullDiff=true. + */ + @Test + public void testFullDiffOnlyMode() throws IOException { + UUID fromSnapshotId = UUID.randomUUID(); + UUID toSnapshotId = UUID.randomUUID(); + SnapshotInfo fromSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap1", fromSnapshotId); + SnapshotInfo toSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap2", toSnapshotId); + Set<String> tablesToLookup = ImmutableSet.of("keyTable"); + TablePrefixInfo tablePrefixInfo = new TablePrefixInfo(ImmutableMap.of("keyTable", "a")); + + Path sstFile = tempDir.resolve("test3.sst"); + Files.createFile(sstFile); + SstFileInfo sstInfo = new SstFileInfo("test3.sst", "key5", "key6", "keyTable"); + Map<Path, Pair<Path, SstFileInfo>> fullDiffResult = new HashMap<>(); + fullDiffResult.put(sstFile, Pair.of(sstFile, sstInfo)); + + try (MockedConstruction<RDBDifferComputer> rdbDifferMock = mockConstruction(RDBDifferComputer.class); + MockedConstruction<FullDiffComputer> fullDiffMock = mockConstruction(FullDiffComputer.class, + (mock, context) -> { + when(mock.computeDeltaFiles(any(), any(), anySet(), any())) + .thenReturn(Optional.of(fullDiffResult)); + })) { + + CompositeDeltaDiffComputer composite = new CompositeDeltaDiffComputer( + omSnapshotManager, activeMetadataManager, deltaDirPath, activityReporter, true, false); + + Optional<Map<Path, Pair<Path, SstFileInfo>>> result = + composite.computeDeltaFiles(fromSnapshot, toSnapshot, tablesToLookup, tablePrefixInfo); + + // Verify RDBDifferComputer was never constructed or called + assertEquals(0, rdbDifferMock.constructed().size(), "RDBDifferComputer should not be constructed"); + + // Verify FullDiffComputer was used + assertTrue(result.isPresent(), "Result should be present"); + FullDiffComputer fullDiffInstance = fullDiffMock.constructed().get(0); + verify(fullDiffInstance, times(1)).computeDeltaFiles(any(), any(), anySet(), any()); + + // Verify only FULL_DIFF status was reported + ArgumentCaptor<SubStatus> statusCaptor = ArgumentCaptor.forClass(SubStatus.class); + verify(activityReporter, times(1)).accept(statusCaptor.capture()); + assertEquals(SubStatus.SST_FILE_DELTA_FULL_DIFF, statusCaptor.getValue()); + + composite.close(); + } + } + + /** + * Tests proper cleanup of both computers. + */ + @Test + public void testCloseCallsBothComputers() throws IOException { + try (MockedConstruction<RDBDifferComputer> rdbDifferMock = mockConstruction(RDBDifferComputer.class); + MockedConstruction<FullDiffComputer> fullDiffMock = mockConstruction(FullDiffComputer.class)) { + + CompositeDeltaDiffComputer composite = new CompositeDeltaDiffComputer( + omSnapshotManager, activeMetadataManager, deltaDirPath, activityReporter, false, false); + + composite.close(); + + // Verify close was called on both + RDBDifferComputer rdbDifferInstance = rdbDifferMock.constructed().get(0); + FullDiffComputer fullDiffInstance = fullDiffMock.constructed().get(0); + + verify(rdbDifferInstance, times(1)).close(); + verify(fullDiffInstance, times(1)).close(); + } + } + + /** + * Tests that nonNativeDiff flag is properly passed to constructor. + * Verifies CompositeDeltaDiffComputer can be created with nonNativeDiff=true. + */ + @Test + public void testNonNativeDiffFlagInConstructor() throws IOException { + try (MockedConstruction<RDBDifferComputer> rdbDifferMock = mockConstruction(RDBDifferComputer.class); + MockedConstruction<FullDiffComputer> fullDiffMock = mockConstruction(FullDiffComputer.class)) { + + // Create with nonNativeDiff = true + CompositeDeltaDiffComputer composite = new CompositeDeltaDiffComputer( + omSnapshotManager, activeMetadataManager, deltaDirPath, activityReporter, false, true); + + // Verify construction succeeds and both computers are created + assertEquals(1, rdbDifferMock.constructed().size(), "RDBDifferComputer should be created"); + assertEquals(1, fullDiffMock.constructed().size(), "FullDiffComputer should be created"); + + composite.close(); + } + } + + /** + * Tests that nonNativeDiff flag works correctly when disabled. + * Verifies CompositeDeltaDiffComputer can be created with nonNativeDiff=false. + */ + @Test + public void testNonNativeDiffDisabled() throws IOException { + try (MockedConstruction<RDBDifferComputer> rdbDifferMock = mockConstruction(RDBDifferComputer.class); + MockedConstruction<FullDiffComputer> fullDiffMock = mockConstruction(FullDiffComputer.class)) { + + // Create with nonNativeDiff = false (default behavior) + CompositeDeltaDiffComputer composite = new CompositeDeltaDiffComputer( + omSnapshotManager, activeMetadataManager, deltaDirPath, activityReporter, false, false); + + // Verify construction succeeds and both computers are created + assertEquals(1, rdbDifferMock.constructed().size(), "RDBDifferComputer should be created"); + assertEquals(1, fullDiffMock.constructed().size(), "FullDiffComputer should be created"); + + composite.close(); + } + } + + /** + * Tests nonNativeDiff mode with computeDeltaFiles - verifies fromSnapshot files are added. + * In nonNativeDiff mode, SST files from fromSnapshot are added to the delta to handle deletes. + */ + @Test + public void testNonNativeDiffComputeDeltaFilesEnabled() throws IOException { + // Given nonNativeDiff is enabled and we have snapshots + UUID fromSnapshotId = UUID.randomUUID(); + UUID toSnapshotId = UUID.randomUUID(); + SnapshotInfo fromSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap1", fromSnapshotId); + SnapshotInfo toSnapshot = createMockSnapshotInfo("vol1", "bucket1", "snap2", toSnapshotId); + Set<String> tablesToLookup = ImmutableSet.of("keyTable"); + TablePrefixInfo tablePrefixInfo = new TablePrefixInfo(ImmutableMap.of("keyTable", "a")); + + // Setup fromSnapshot SST files + Path fromDbPath = tempDir.resolve("fromDb"); + Files.createDirectories(fromDbPath); + Path fromSstFile1 = fromDbPath.resolve("000001.sst"); + Path fromSstFile2 = fromDbPath.resolve("000002.sst"); + Files.createFile(fromSstFile1); + Files.createFile(fromSstFile2); + + SstFileInfo fromSstInfo1 = new SstFileInfo("000001", "a/key1", "a/key100", "keyTable"); + SstFileInfo fromSstInfo2 = new SstFileInfo("000002", "a/key101", "a/key200", "keyTable"); + Set<SstFileInfo> fromSnapshotSstFiles = ImmutableSet.of(fromSstInfo1, fromSstInfo2); + + // Mock fromSnapshot + OmSnapshot fromSnap = org.mockito.Mockito.mock(OmSnapshot.class); + OMMetadataManager fromMetaMgr = org.mockito.Mockito.mock(OMMetadataManager.class); + RDBStore fromRdbStore = org.mockito.Mockito.mock(RDBStore.class); + when(fromSnap.getMetadataManager()).thenReturn(fromMetaMgr); + when(fromMetaMgr.getStore()).thenReturn(fromRdbStore); + when(fromRdbStore.getDbLocation()).thenReturn(fromDbPath.toFile()); + + @SuppressWarnings("unchecked") + UncheckedAutoCloseableSupplier<OmSnapshot> fromSnapSupplier = + (UncheckedAutoCloseableSupplier<OmSnapshot>) org.mockito.Mockito.mock(UncheckedAutoCloseableSupplier.class); Review Comment: Inconsistent use of `mock()` method. Lines 605-607 and 614 use the fully qualified `org.mockito.Mockito.mock()` instead of the already imported static method `mock()` (visible from the static imports at the top). Use `mock(OmSnapshot.class)` instead of `org.mockito.Mockito.mock(OmSnapshot.class)` for consistency with the rest of the test file. ```suggestion OmSnapshot fromSnap = mock(OmSnapshot.class); OMMetadataManager fromMetaMgr = mock(OMMetadataManager.class); RDBStore fromRdbStore = mock(RDBStore.class); when(fromSnap.getMetadataManager()).thenReturn(fromMetaMgr); when(fromMetaMgr.getStore()).thenReturn(fromRdbStore); when(fromRdbStore.getDbLocation()).thenReturn(fromDbPath.toFile()); @SuppressWarnings("unchecked") UncheckedAutoCloseableSupplier<OmSnapshot> fromSnapSupplier = (UncheckedAutoCloseableSupplier<OmSnapshot>) mock(UncheckedAutoCloseableSupplier.class); ``` ########## hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/CompositeDeltaDiffComputer.java: ########## @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot.diff.delta; + +import static org.apache.hadoop.ozone.om.snapshot.diff.delta.FullDiffComputer.getSSTFileSetForSnapshot; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.utils.db.TablePrefixInfo; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; +import org.apache.ozone.rocksdb.util.SstFileInfo; +import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * CompositeDeltaDiffComputer is responsible for computing the delta file + * differences between two snapshots, utilizing different strategies such + * as partial differ computation and full differ computation. + * + * It serves as an orchestrator to decide whether to perform a full diff + * or a more efficient partial diff, and handles fallback mechanisms if + * the chosen method fails. + * + * The class leverages two main difference computation strategies: + * - {@code RDBDifferComputer} for partial diff computation + * - {@code FullDiffComputer} for exhaustive diff + * + * This class also includes support for handling non-native diff scenarios + * through additional processing of input files from the "from" snapshot + * when native RocksDB tools are not used. + * + * Inherits from {@code FileLinkDeltaFileComputer} and implements the + * functionality for computing delta files and resource management. + */ +public class CompositeDeltaDiffComputer extends FileLinkDeltaFileComputer { + + private static final Logger LOG = LoggerFactory.getLogger(CompositeDeltaDiffComputer.class); + + private final RDBDifferComputer differComputer; + private final FullDiffComputer fullDiffComputer; + private final boolean nonNativeDiff; + + public CompositeDeltaDiffComputer(OmSnapshotManager snapshotManager, + OMMetadataManager activeMetadataManager, Path deltaDirPath, + Consumer<SnapshotDiffResponse.SubStatus> activityReporter, boolean fullDiff, + boolean nonNativeDiff) throws IOException { + super(snapshotManager, activeMetadataManager, deltaDirPath, activityReporter); + differComputer = fullDiff ? null : new RDBDifferComputer(snapshotManager, activeMetadataManager, + deltaDirPath.resolve("rdbDiffer"), activityReporter); + fullDiffComputer = new FullDiffComputer(snapshotManager, activeMetadataManager, + deltaDirPath.resolve("fullDiff"), activityReporter); + this.nonNativeDiff = nonNativeDiff; + } + + @Override + Optional<Map<Path, Pair<Path, SstFileInfo>>> computeDeltaFiles(SnapshotInfo fromSnapshotInfo, + SnapshotInfo toSnapshotInfo, Set<String> tablesToLookup, TablePrefixInfo tablePrefixInfo) throws IOException { + Map<Path, Pair<Path, SstFileInfo>> deltaFiles = null; + try { + if (differComputer != null) { + updateActivity(SnapshotDiffResponse.SubStatus.SST_FILE_DELTA_DAG_WALK); + deltaFiles = differComputer.computeDeltaFiles(fromSnapshotInfo, toSnapshotInfo, tablesToLookup, + tablePrefixInfo).orElse(null); + } + } catch (Exception e) { + LOG.warn("Falling back to full diff.", e); + } + if (deltaFiles == null) { + updateActivity(SnapshotDiffResponse.SubStatus.SST_FILE_DELTA_FULL_DIFF); + deltaFiles = fullDiffComputer.computeDeltaFiles(fromSnapshotInfo, toSnapshotInfo, tablesToLookup, + tablePrefixInfo).orElse(null); + if (deltaFiles == null) { + // FileLinkDeltaFileComputer would throw an exception in this case. + return Optional.empty(); + } + } + // Workaround to handle deletes if native rocksDb tool for reading + // tombstone is not loaded. + // When performing non native diff, input files of from snapshot needs to be added. + if (nonNativeDiff) { + try (UncheckedAutoCloseableSupplier<OmSnapshot> fromSnapshot = getSnapshot(fromSnapshotInfo)) { + Set<SstFileInfo> fromSnapshotFiles = getSSTFileSetForSnapshot(fromSnapshot.get(), tablesToLookup, + tablePrefixInfo); + Path fromSnapshotPath = fromSnapshot.get().getMetadataManager().getStore().getDbLocation() + .getAbsoluteFile().toPath(); Review Comment: Access of [element](1) annotated with VisibleForTesting found in production code. ```suggestion Path fromSnapshotPath = fromSnapshot.get().getDbLocation().getAbsoluteFile().toPath(); ``` ########## hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/CompositeDeltaDiffComputer.java: ########## @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot.diff.delta; + +import static org.apache.hadoop.ozone.om.snapshot.diff.delta.FullDiffComputer.getSSTFileSetForSnapshot; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.utils.db.TablePrefixInfo; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; +import org.apache.ozone.rocksdb.util.SstFileInfo; +import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * CompositeDeltaDiffComputer is responsible for computing the delta file + * differences between two snapshots, utilizing different strategies such + * as partial differ computation and full differ computation. + * + * It serves as an orchestrator to decide whether to perform a full diff + * or a more efficient partial diff, and handles fallback mechanisms if + * the chosen method fails. + * + * The class leverages two main difference computation strategies: + * - {@code RDBDifferComputer} for partial diff computation + * - {@code FullDiffComputer} for exhaustive diff + * + * This class also includes support for handling non-native diff scenarios + * through additional processing of input files from the "from" snapshot + * when native RocksDB tools are not used. + * + * Inherits from {@code FileLinkDeltaFileComputer} and implements the + * functionality for computing delta files and resource management. + */ +public class CompositeDeltaDiffComputer extends FileLinkDeltaFileComputer { + + private static final Logger LOG = LoggerFactory.getLogger(CompositeDeltaDiffComputer.class); + + private final RDBDifferComputer differComputer; + private final FullDiffComputer fullDiffComputer; + private final boolean nonNativeDiff; + + public CompositeDeltaDiffComputer(OmSnapshotManager snapshotManager, + OMMetadataManager activeMetadataManager, Path deltaDirPath, + Consumer<SnapshotDiffResponse.SubStatus> activityReporter, boolean fullDiff, + boolean nonNativeDiff) throws IOException { + super(snapshotManager, activeMetadataManager, deltaDirPath, activityReporter); + differComputer = fullDiff ? null : new RDBDifferComputer(snapshotManager, activeMetadataManager, + deltaDirPath.resolve("rdbDiffer"), activityReporter); + fullDiffComputer = new FullDiffComputer(snapshotManager, activeMetadataManager, + deltaDirPath.resolve("fullDiff"), activityReporter); + this.nonNativeDiff = nonNativeDiff; + } + + @Override + Optional<Map<Path, Pair<Path, SstFileInfo>>> computeDeltaFiles(SnapshotInfo fromSnapshotInfo, + SnapshotInfo toSnapshotInfo, Set<String> tablesToLookup, TablePrefixInfo tablePrefixInfo) throws IOException { + Map<Path, Pair<Path, SstFileInfo>> deltaFiles = null; + try { + if (differComputer != null) { + updateActivity(SnapshotDiffResponse.SubStatus.SST_FILE_DELTA_DAG_WALK); + deltaFiles = differComputer.computeDeltaFiles(fromSnapshotInfo, toSnapshotInfo, tablesToLookup, + tablePrefixInfo).orElse(null); + } + } catch (Exception e) { + LOG.warn("Falling back to full diff.", e); + } + if (deltaFiles == null) { + updateActivity(SnapshotDiffResponse.SubStatus.SST_FILE_DELTA_FULL_DIFF); + deltaFiles = fullDiffComputer.computeDeltaFiles(fromSnapshotInfo, toSnapshotInfo, tablesToLookup, + tablePrefixInfo).orElse(null); + if (deltaFiles == null) { + // FileLinkDeltaFileComputer would throw an exception in this case. + return Optional.empty(); + } + } + // Workaround to handle deletes if native rocksDb tool for reading + // tombstone is not loaded. + // When performing non native diff, input files of from snapshot needs to be added. + if (nonNativeDiff) { + try (UncheckedAutoCloseableSupplier<OmSnapshot> fromSnapshot = getSnapshot(fromSnapshotInfo)) { + Set<SstFileInfo> fromSnapshotFiles = getSSTFileSetForSnapshot(fromSnapshot.get(), tablesToLookup, + tablePrefixInfo); + Path fromSnapshotPath = fromSnapshot.get().getMetadataManager().getStore().getDbLocation() + .getAbsoluteFile().toPath(); Review Comment: Access of [element](1) annotated with VisibleForTesting found in production code. ```suggestion Path fromSnapshotPath = fromSnapshot.get().getSnapshotPath(); ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
