Murtadha Hubail has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/3331
Change subject: [NO ISSUE][STO] Ensure Invalid Merged Components Are Deleted
......................................................................
[NO ISSUE][STO] Ensure Invalid Merged Components Are Deleted
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When cleaning up an index's invalid components, check the
components end sequence in addition to the start sequence
to ensure invalid merged components with are cleaned too.
- Add test case.
Change-Id: I80d72b1b614718e2e283bc72a874c140d178d1e0
---
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
M
asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
2 files changed, 22 insertions(+), 16 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/31/3331/1
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
index 54ae683..00d2d3d 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
@@ -19,6 +19,7 @@
package org.apache.asterix.test.storage;
import java.io.File;
+import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -127,19 +128,9 @@
String invalidComponentRange = invalidComponentId +
AbstractLSMIndexFileManager.DELIMITER + invalidComponentId;
FileReference indexDirRef = ncAppCtx.getIoManager().resolve(indexPath);
String indexDir = indexDirRef.getFile().getAbsolutePath();
- // create the invalid component files
- Path btreePath = Paths.get(indexDir, invalidComponentRange +
AbstractLSMIndexFileManager.DELIMITER
- + AbstractLSMIndexFileManager.BTREE_SUFFIX);
- Path filterPath = Paths.get(indexDir, invalidComponentRange +
AbstractLSMIndexFileManager.DELIMITER
- + AbstractLSMIndexFileManager.BLOOM_FILTER_SUFFIX);
- Files.createFile(btreePath);
- Files.createFile(filterPath);
-
- // clean up the index partition
- localResourceRepository.cleanup(lr.getPartition());
- // ensure that the invalid component was deleted
- Assert.assertFalse(btreePath.toFile().exists());
- Assert.assertFalse(filterPath.toFile().exists());
+ ensureInvalidComponentDeleted(indexDir, invalidComponentRange,
localResourceRepository, lr);
+ String invalidMergeComponentRange = "0" +
AbstractLSMIndexFileManager.DELIMITER + invalidComponentId;
+ ensureInvalidComponentDeleted(indexDir, invalidMergeComponentRange,
localResourceRepository, lr);
// ensure that valid components still exist
// find index valid component timestamp from checkpoint
@@ -179,4 +170,17 @@
Assert.assertFalse(indexMetadataFile.exists());
Assert.assertFalse(indexMetadataMaskFile.exists());
}
+
+ private void ensureInvalidComponentDeleted(String indexDir, String
componentSeq,
+ PersistentLocalResourceRepository localResourceRepository,
DatasetLocalResource lr) throws IOException {
+ Path btreePath = Paths.get(indexDir,
+ componentSeq + AbstractLSMIndexFileManager.DELIMITER +
AbstractLSMIndexFileManager.BTREE_SUFFIX);
+ Path filterPath = Paths.get(indexDir,
+ componentSeq + AbstractLSMIndexFileManager.DELIMITER +
AbstractLSMIndexFileManager.BLOOM_FILTER_SUFFIX);
+ Files.createFile(btreePath);
+ Files.createFile(filterPath);
+ localResourceRepository.cleanup(lr.getPartition());
+ Assert.assertFalse(btreePath.toFile().exists());
+ Assert.assertFalse(filterPath.toFile().exists());
+ }
}
diff --git
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index 8f870c0..aef7bbd 100644
---
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -497,10 +497,12 @@
}
final long validComponentSequence =
getIndexCheckpointManager(index).getValidComponentSequence();
for (File componentFile : indexComponentFiles) {
- // delete any file with start sequence > valid component sequence
+ // delete any file with start or end sequence > valid component
sequence
final long fileStart =
IndexComponentFileReference.of(componentFile.getName()).getSequenceStart();
- if (fileStart > validComponentSequence) {
- LOGGER.info(() -> "Deleting invalid component file: " +
componentFile.getAbsolutePath());
+ final long fileEnd =
IndexComponentFileReference.of(componentFile.getName()).getSequenceEnd();
+ if (fileStart > validComponentSequence || fileEnd >
validComponentSequence) {
+ LOGGER.warn(() -> "Deleting invalid component file " +
componentFile.getAbsolutePath()
+ + " based on valid sequence " +
validComponentSequence);
Files.delete(componentFile.toPath());
}
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/3331
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I80d72b1b614718e2e283bc72a874c140d178d1e0
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: stabilization-f69489
Gerrit-Owner: Murtadha Hubail <[email protected]>
