swamirishi commented on code in PR #9140:
URL: https://github.com/apache/ozone/pull/9140#discussion_r2440764758
##########
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLocalDataManager.java:
##########
@@ -135,6 +165,376 @@ public void tearDown() throws Exception {
}
}
+ private String getReadLockMessageAcquire(UUID snapshotId) {
+ return READ_LOCK_MESSAGE_ACQUIRE + " " +
FlatResource.SNAPSHOT_LOCAL_DATA_LOCK + " " + snapshotId;
+ }
+
+ private String getReadLockMessageRelease(UUID snapshotId) {
+ return READ_LOCK_MESSAGE_UNLOCK + " " +
FlatResource.SNAPSHOT_LOCAL_DATA_LOCK + " " + snapshotId;
+ }
+
+ private String getWriteLockMessageAcquire(UUID snapshotId) {
+ return WRITE_LOCK_MESSAGE_ACQUIRE + " " +
FlatResource.SNAPSHOT_LOCAL_DATA_LOCK + " " + snapshotId;
+ }
+
+ private String getWriteLockMessageRelease(UUID snapshotId) {
+ return WRITE_LOCK_MESSAGE_UNLOCK + " " +
FlatResource.SNAPSHOT_LOCAL_DATA_LOCK + " " + snapshotId;
+ }
+
+ private HierarchicalResourceLock getHierarchicalResourceLock(FlatResource
resource, String key, boolean isWriteLock) {
+ return new HierarchicalResourceLock() {
+ @Override
+ public boolean isLockAcquired() {
+ return true;
+ }
+
+ @Override
+ public void close() {
+ if (isWriteLock) {
+ lockCapturor.add(WRITE_LOCK_MESSAGE_UNLOCK + " " + resource + " " +
key);
+ } else {
+ lockCapturor.add(READ_LOCK_MESSAGE_UNLOCK + " " + resource + " " +
key);
+ }
+ }
+ };
+ }
+
+ private void mockLockManager() throws IOException {
+ lockCapturor.clear();
+ reset(lockManager);
+ when(lockManager.acquireReadLock(any(FlatResource.class), anyString()))
+ .thenAnswer(i -> {
+ lockCapturor.add(READ_LOCK_MESSAGE_ACQUIRE + " " + i.getArgument(0)
+ " " + i.getArgument(1));
+ return getHierarchicalResourceLock(i.getArgument(0),
i.getArgument(1), false);
+ });
+ when(lockManager.acquireWriteLock(any(FlatResource.class), anyString()))
+ .thenAnswer(i -> {
+ lockCapturor.add(WRITE_LOCK_MESSAGE_ACQUIRE + " " + i.getArgument(0)
+ " " + i.getArgument(1));
+ return getHierarchicalResourceLock(i.getArgument(0),
i.getArgument(1), true);
+ });
+ }
+
+ private List<UUID> createSnapshotLocalData(OmSnapshotLocalDataManager
snapshotLocalDataManager,
+ int numberOfSnapshots) throws IOException {
+ SnapshotInfo previousSnapshotInfo = null;
+ int counter = 0;
+ Map<String, List<LiveFileMetaData>> liveFileMetaDataMap = new HashMap<>();
+ liveFileMetaDataMap.put(KEY_TABLE,
+ Lists.newArrayList(createMockLiveFileMetaData("file1.sst", KEY_TABLE,
"key1", "key2")));
+ liveFileMetaDataMap.put(FILE_TABLE,
Lists.newArrayList(createMockLiveFileMetaData("file2.sst", FILE_TABLE, "key1",
+ "key2")));
+ liveFileMetaDataMap.put(DIRECTORY_TABLE,
Lists.newArrayList(createMockLiveFileMetaData("file2.sst",
+ DIRECTORY_TABLE, "key1", "key2")));
+ liveFileMetaDataMap.put("col1",
Lists.newArrayList(createMockLiveFileMetaData("file2.sst", "col1", "key1",
+ "key2")));
+ List<UUID> snapshotIds = new ArrayList<>();
+ for (int i = 0; i < numberOfSnapshots; i++) {
+ UUID snapshotId = UUID.randomUUID();
+ SnapshotInfo snapshotInfo = createMockSnapshotInfo(snapshotId,
previousSnapshotInfo == null ? null
+ : previousSnapshotInfo.getSnapshotId());
+ mockSnapshotStore(snapshotId, liveFileMetaDataMap.values().stream()
+ .flatMap(Collection::stream).collect(Collectors.toList()));
+ snapshotLocalDataManager.createNewOmSnapshotLocalDataFile(snapshotStore,
snapshotInfo);
+ previousSnapshotInfo = snapshotInfo;
+ for (Map.Entry<String, List<LiveFileMetaData>> tableEntry :
liveFileMetaDataMap.entrySet()) {
+ String table = tableEntry.getKey();
+ tableEntry.getValue().add(createMockLiveFileMetaData("file" +
counter++ + ".sst", table, "key1", "key4"));
+ }
+ snapshotIds.add(snapshotId);
+ }
+ return snapshotIds;
+ }
+
+ private void mockSnapshotStore(UUID snapshotId, List<LiveFileMetaData>
sstFiles) throws RocksDatabaseException {
+ // Setup snapshot store mock
+ File snapshotDbLocation =
OmSnapshotManager.getSnapshotPath(omMetadataManager, snapshotId).toFile();
+ assertTrue(snapshotDbLocation.exists() || snapshotDbLocation.mkdirs());
+
+ when(snapshotStore.getDbLocation()).thenReturn(snapshotDbLocation);
+ RocksDatabase rocksDatabase = mock(RocksDatabase.class);
+ when(snapshotStore.getDb()).thenReturn(rocksDatabase);
+ when(rocksDatabase.getLiveFilesMetaData()).thenReturn(sstFiles);
+ }
+
+ /**
+ * Checks lock orders taken i.e. while reading a snapshot against the
previous snapshot.
+ * Depending on read or write locks are acquired on the snapshotId and read
lock is acquired on the previous
+ * snapshot. Once the instance is closed the read lock on previous snapshot
is released followed by releasing the
+ * lock on the snapshotId.
+ * @param read
+ * @throws IOException
+ */
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testLockOrderingAgainstAnotherSnapshot(boolean read) throws
IOException {
+ localDataManager = new OmSnapshotLocalDataManager(omMetadataManager);
+ List<UUID> snapshotIds = new ArrayList<>();
+ snapshotIds.add(null);
+ snapshotIds.addAll(createSnapshotLocalData(localDataManager, 20));
+ for (int start = 0; start < snapshotIds.size(); start++) {
+ for (int end = start + 1; end < snapshotIds.size(); end++) {
+ UUID startSnapshotId = snapshotIds.get(start);
+ UUID endSnapshotId = snapshotIds.get(end);
+ lockCapturor.clear();
+ int logCaptorIdx = 0;
+ try (ReadableOmSnapshotLocalDataProvider omSnapshotLocalDataProvider =
+ read ? localDataManager.getOmSnapshotLocalData(endSnapshotId,
startSnapshotId) :
+
localDataManager.getWritableOmSnapshotLocalData(endSnapshotId,
startSnapshotId)) {
+ OmSnapshotLocalData snapshotLocalData =
omSnapshotLocalDataProvider.getSnapshotLocalData();
+ OmSnapshotLocalData previousSnapshot =
omSnapshotLocalDataProvider.getPreviousSnapshotLocalData();
+ assertEquals(endSnapshotId, snapshotLocalData.getSnapshotId());
+ if (startSnapshotId == null) {
+ assertNull(previousSnapshot);
+ assertNull(snapshotLocalData.getPreviousSnapshotId());
+ continue;
+ }
+ assertEquals(startSnapshotId, previousSnapshot.getSnapshotId());
+ assertEquals(startSnapshotId,
snapshotLocalData.getPreviousSnapshotId());
+ if (read) {
+ assertEquals(getReadLockMessageAcquire(endSnapshotId),
lockCapturor.get(logCaptorIdx++));
+ } else {
+ assertEquals(getWriteLockMessageAcquire(endSnapshotId),
lockCapturor.get(logCaptorIdx++));
+ }
+ int idx = end - 1;
+ UUID previousSnapId = snapshotIds.get(idx--);
+ assertEquals(getReadLockMessageAcquire(previousSnapId),
lockCapturor.get(logCaptorIdx++));
+ while (idx >= start) {
+ UUID prevPrevSnapId = snapshotIds.get(idx--);
+ assertEquals(getReadLockMessageAcquire(prevPrevSnapId),
lockCapturor.get(logCaptorIdx++));
+ assertEquals(getReadLockMessageRelease(previousSnapId),
lockCapturor.get(logCaptorIdx++));
+ previousSnapId = prevPrevSnapId;
+ }
+ }
+ assertEquals(getReadLockMessageRelease(startSnapshotId),
lockCapturor.get(logCaptorIdx++));
+ if (read) {
+ assertEquals(getReadLockMessageRelease(endSnapshotId),
lockCapturor.get(logCaptorIdx++));
+ } else {
+ assertEquals(getWriteLockMessageRelease(endSnapshotId),
lockCapturor.get(logCaptorIdx++));
+ }
+ assertEquals(lockCapturor.size(), logCaptorIdx);
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testVersionLockResolution(boolean read) throws IOException {
+ localDataManager = new OmSnapshotLocalDataManager(omMetadataManager);
+ List<UUID> snapshotIds = createSnapshotLocalData(localDataManager, 5);
+ for (int snapIdx = 0; snapIdx < snapshotIds.size(); snapIdx++) {
+ UUID snapId = snapshotIds.get(snapIdx);
+ UUID expectedPreviousSnapId = snapIdx - 1 >= 0 ? snapshotIds.get(snapIdx
- 1) : null;
+ lockCapturor.clear();
+ int logCaptorIdx = 0;
+ try (ReadableOmSnapshotLocalDataProvider omSnapshotLocalDataProvider =
+ read ? localDataManager.getOmSnapshotLocalData(snapId) :
+ localDataManager.getWritableOmSnapshotLocalData(snapId)) {
+ OmSnapshotLocalData snapshotLocalData =
omSnapshotLocalDataProvider.getSnapshotLocalData();
+ OmSnapshotLocalData previousSnapshot =
omSnapshotLocalDataProvider.getPreviousSnapshotLocalData();
+ assertEquals(snapId, snapshotLocalData.getSnapshotId());
+ assertEquals(expectedPreviousSnapId, previousSnapshot == null ? null :
+ previousSnapshot.getSnapshotId());
+ if (read) {
+ assertEquals(getReadLockMessageAcquire(snapId),
lockCapturor.get(logCaptorIdx++));
+ } else {
+ assertEquals(getWriteLockMessageAcquire(snapId),
lockCapturor.get(logCaptorIdx++));
+ }
+ if (expectedPreviousSnapId != null) {
+ assertEquals(getReadLockMessageAcquire(expectedPreviousSnapId),
lockCapturor.get(logCaptorIdx++));
+ }
+ }
+ if (expectedPreviousSnapId != null) {
+ assertEquals(getReadLockMessageRelease(expectedPreviousSnapId),
lockCapturor.get(logCaptorIdx++));
+ }
+ if (read) {
+ assertEquals(getReadLockMessageRelease(snapId),
lockCapturor.get(logCaptorIdx++));
+ } else {
+ assertEquals(getWriteLockMessageRelease(snapId),
lockCapturor.get(logCaptorIdx++));
+ }
+ assertEquals(lockCapturor.size(), logCaptorIdx);
+ }
+ }
+
+ @Test
+ public void
testWriteVersionAdditionValidationWithoutPreviousSnapshotVersionExisting()
throws IOException {
+ localDataManager = new OmSnapshotLocalDataManager(omMetadataManager);
+ List<UUID> snapshotIds = createSnapshotLocalData(localDataManager, 2);
+ UUID snapId = snapshotIds.get(1);
+ try (WritableOmSnapshotLocalDataProvider omSnapshotLocalDataProvider =
+ localDataManager.getWritableOmSnapshotLocalData(snapId)) {
+ OmSnapshotLocalData snapshotLocalData =
omSnapshotLocalDataProvider.getSnapshotLocalData();
+
snapshotLocalData.addVersionSSTFileInfos(Lists.newArrayList(createMockLiveFileMetaData("file1.sst",
KEY_TABLE,
+ "key1", "key2")), 3);
+
+ IOException ex = assertThrows(IOException.class,
omSnapshotLocalDataProvider::commit);
+ System.out.println(ex.getMessage());
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
