abdullah alamoudi has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/2822
Change subject: [ASTERIXDB-2414][STO] Remove deleted component files from
buffer cache
......................................................................
[ASTERIXDB-2414][STO] Remove deleted component files from buffer cache
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When activating an LSM index, we remove files of components
that were merged into a bigger component but not cleaned up yet.
- However, we sometimes leave a file reference mapped in the buffer
cache even when the file is removed from disk.
- This change ensures that all files are removed from the buffer
cache as well.
Change-Id: If0f11bc222662e4b50c1b47b1dfa6b30d1463b2e
---
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
5 files changed, 22 insertions(+), 22 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/22/2822/1
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
index 54bc1fe..dbfea6f 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.lsm.btree.impls;
-import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.ArrayList;
@@ -148,9 +147,9 @@
&& (!hasBloomFilter ||
(currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0
&&
currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0))) {
// Invalid files are completely contained in last interval.
- IoUtil.delete(new File(currentBTree.fullPath));
+ delete(btreeFactory.getBufferCache(), currentBTree.fullPath);
if (hasBloomFilter) {
- IoUtil.delete(new File(currentBloomFilter.fullPath));
+ delete(btreeFactory.getBufferCache(),
currentBloomFilter.fullPath);
}
} else {
// This scenario should not be possible.
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
index f61d783..644abcf 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
@@ -18,7 +18,6 @@
*/
package org.apache.hyracks.storage.am.lsm.btree.impls;
-import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.file.Files;
@@ -153,9 +152,9 @@
&&
currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0
&&
currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0) {
// Invalid files are completely contained in last interval.
- IoUtil.delete(new File(currentBTree.fullPath));
- IoUtil.delete(new File(currentBuddyBTree.fullPath));
- IoUtil.delete(new File(currentBloomFilter.fullPath));
+ delete(treeFactory.getBufferCache(), currentBTree.fullPath);
+ delete(treeFactory.getBufferCache(),
currentBuddyBTree.fullPath);
+ delete(treeFactory.getBufferCache(),
currentBloomFilter.fullPath);
} else {
// This scenario should not be possible.
throw
HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir);
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
index 59a919b..b57faba 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.lsm.common.impls;
-import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.text.Format;
@@ -141,11 +140,13 @@
allFiles.add(new ComparableFileName(fileRef));
continue;
}
- TreeIndexState idxState =
isValidTreeIndex(treeFactory.createIndexInstance(fileRef));
+ IBufferCache bufferCache = treeFactory.getBufferCache();
+ ITreeIndex treeIndex = treeFactory.createIndexInstance(fileRef);
+ TreeIndexState idxState = isValidTreeIndex(treeIndex);
if (idxState == TreeIndexState.VALID) {
allFiles.add(new ComparableFileName(fileRef));
} else if (idxState == TreeIndexState.INVALID) {
- fileRef.delete();
+ bufferCache.deleteFile(fileRef);
}
}
}
@@ -178,8 +179,7 @@
if (groundTruth.contains(file)) {
validFiles.add(cmpFileName);
} else {
- File invalidFile = new File(cmpFileName.fullPath);
- IoUtil.delete(invalidFile);
+ delete(treeFactory.getBufferCache(), cmpFileName.fullPath);
}
}
}
@@ -252,7 +252,7 @@
// The current file is completely contained in the interval of
the
// last file. Thus the last file must contain at least as much
information
// as the current file, so delete the current file.
- current.fileRef.delete();
+ delete(treeFactory.getBufferCache(), current.fullPath);
} else {
// This scenario should not be possible since timestamps are
monotonically increasing.
throw
HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir);
@@ -387,6 +387,11 @@
}
}
+ protected void delete(IBufferCache bufferCache, String fullPath) throws
HyracksDataException {
+ FileReference fileRef = ioManager.resolveAbsolutePath(fullPath);
+ bufferCache.deleteFile(fileRef);
+ }
+
protected FilenameFilter getCompoundFilter(final FilenameFilter filter1,
final FilenameFilter filter2) {
return (dir, name) -> filter1.accept(dir, name) && filter2.accept(dir,
name);
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
index c4a2f73..4742c02 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.impls;
-import java.io.File;
import java.io.FilenameFilter;
import java.util.ArrayList;
import java.util.Collections;
@@ -31,7 +30,6 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
-import org.apache.hyracks.api.util.IoUtil;
import
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory;
import
org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
@@ -158,9 +156,9 @@
&&
currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0
&&
currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0) {
// Invalid files are completely contained in last interval.
- IoUtil.delete(new File(currentDeletedKeysBTree.fullPath));
- IoUtil.delete(new File(currentDictBTree.fullPath));
- IoUtil.delete(new File(currentBloomFilter.fullPath));
+ delete(treeFactory.getBufferCache(),
currentDeletedKeysBTree.fullPath);
+ delete(treeFactory.getBufferCache(),
currentDictBTree.fullPath);
+ delete(treeFactory.getBufferCache(),
currentBloomFilter.fullPath);
} else {
// This scenario should not be possible.
throw
HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir);
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
index 001228c..571086d 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.lsm.rtree.impls;
-import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.file.Files;
@@ -155,9 +154,9 @@
&&
currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0
&&
currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0) {
// Invalid files are completely contained in last interval.
- IoUtil.delete(new File(currentRTree.fullPath));
- IoUtil.delete(new File(currentBTree.fullPath));
- IoUtil.delete(new File(currentBloomFilter.fullPath));
+ delete(treeFactory.getBufferCache(), currentRTree.fullPath);
+ delete(treeFactory.getBufferCache(), currentBTree.fullPath);
+ delete(treeFactory.getBufferCache(),
currentBloomFilter.fullPath);
} else {
// This scenario should not be possible.
throw
HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir);
--
To view, visit https://asterix-gerrit.ics.uci.edu/2822
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If0f11bc222662e4b50c1b47b1dfa6b30d1463b2e
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <[email protected]>
