Murtadha Hubail has submitted this change and it was merged. Change subject: [NO ISSUE][STO] Ensure Invalid Merged Components Are Deleted ......................................................................
[NO ISSUE][STO] Ensure Invalid Merged Components Are Deleted - user model changes: no - storage format changes: no - interface changes: no Details: - When cleaning up an index's invalid components, check the components end sequence in addition to the start sequence to ensure invalid merged components with are cleaned too. - Add test case. Change-Id: I80d72b1b614718e2e283bc72a874c140d178d1e0 Reviewed-on: https://asterix-gerrit.ics.uci.edu/3331 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> --- M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java 2 files changed, 22 insertions(+), 16 deletions(-) Approvals: Anon. E. Moose #1000171: Till Westmann: Looks good to me, approved Jenkins: Verified; No violations found; Verified diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java index 54ae683..00d2d3d 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java @@ -19,6 +19,7 @@ package org.apache.asterix.test.storage; import java.io.File; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -127,19 +128,9 @@ String invalidComponentRange = invalidComponentId + AbstractLSMIndexFileManager.DELIMITER + invalidComponentId; FileReference indexDirRef = ncAppCtx.getIoManager().resolve(indexPath); String indexDir = indexDirRef.getFile().getAbsolutePath(); - // create the invalid component files - Path btreePath = Paths.get(indexDir, invalidComponentRange + AbstractLSMIndexFileManager.DELIMITER - + AbstractLSMIndexFileManager.BTREE_SUFFIX); - Path filterPath = Paths.get(indexDir, invalidComponentRange + AbstractLSMIndexFileManager.DELIMITER - + AbstractLSMIndexFileManager.BLOOM_FILTER_SUFFIX); - Files.createFile(btreePath); - Files.createFile(filterPath); - - // clean up the index partition - localResourceRepository.cleanup(lr.getPartition()); - // ensure that the invalid component was deleted - Assert.assertFalse(btreePath.toFile().exists()); - Assert.assertFalse(filterPath.toFile().exists()); + ensureInvalidComponentDeleted(indexDir, invalidComponentRange, localResourceRepository, lr); + String invalidMergeComponentRange = "0" + AbstractLSMIndexFileManager.DELIMITER + invalidComponentId; + ensureInvalidComponentDeleted(indexDir, invalidMergeComponentRange, localResourceRepository, lr); // ensure that valid components still exist // find index valid component timestamp from checkpoint @@ -179,4 +170,17 @@ Assert.assertFalse(indexMetadataFile.exists()); Assert.assertFalse(indexMetadataMaskFile.exists()); } + + private void ensureInvalidComponentDeleted(String indexDir, String componentSeq, + PersistentLocalResourceRepository localResourceRepository, DatasetLocalResource lr) throws IOException { + Path btreePath = Paths.get(indexDir, + componentSeq + AbstractLSMIndexFileManager.DELIMITER + AbstractLSMIndexFileManager.BTREE_SUFFIX); + Path filterPath = Paths.get(indexDir, + componentSeq + AbstractLSMIndexFileManager.DELIMITER + AbstractLSMIndexFileManager.BLOOM_FILTER_SUFFIX); + Files.createFile(btreePath); + Files.createFile(filterPath); + localResourceRepository.cleanup(lr.getPartition()); + Assert.assertFalse(btreePath.toFile().exists()); + Assert.assertFalse(filterPath.toFile().exists()); + } } diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java index 8f870c0..aef7bbd 100644 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java +++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java @@ -497,10 +497,12 @@ } final long validComponentSequence = getIndexCheckpointManager(index).getValidComponentSequence(); for (File componentFile : indexComponentFiles) { - // delete any file with start sequence > valid component sequence + // delete any file with start or end sequence > valid component sequence final long fileStart = IndexComponentFileReference.of(componentFile.getName()).getSequenceStart(); - if (fileStart > validComponentSequence) { - LOGGER.info(() -> "Deleting invalid component file: " + componentFile.getAbsolutePath()); + final long fileEnd = IndexComponentFileReference.of(componentFile.getName()).getSequenceEnd(); + if (fileStart > validComponentSequence || fileEnd > validComponentSequence) { + LOGGER.warn(() -> "Deleting invalid component file " + componentFile.getAbsolutePath() + + " based on valid sequence " + validComponentSequence); Files.delete(componentFile.toPath()); } } -- To view, visit https://asterix-gerrit.ics.uci.edu/3331 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I80d72b1b614718e2e283bc72a874c140d178d1e0 Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: stabilization-f69489 Gerrit-Owner: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]>
