abdullah alamoudi has submitted this change and it was merged. Change subject: [ASTERIXDB-2414][STO] Remove deleted component files from buffer cache ......................................................................
[ASTERIXDB-2414][STO] Remove deleted component files from buffer cache - user model changes: no - storage format changes: no - interface changes: no Details: - When activating an LSM index, we remove files of components that were merged into a bigger component but not cleaned up yet. - However, we sometimes leave a file reference mapped in the buffer cache even when the file is removed from disk. - This change ensures that all files are removed from the buffer cache as well. Change-Id: If0f11bc222662e4b50c1b47b1dfa6b30d1463b2e Reviewed-on: https://asterix-gerrit.ics.uci.edu/2822 Sonar-Qube: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Contrib: Jenkins <[email protected]> Reviewed-by: abdullah alamoudi <[email protected]> --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java M hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMIndexFileManagerTest.java 8 files changed, 77 insertions(+), 50 deletions(-) Approvals: Anon. E. Moose #1000171: abdullah alamoudi: Looks good to me, approved Jenkins: Verified; No violations found; ; Verified diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java index 54bc1fe..7fbde73 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java @@ -19,7 +19,6 @@ package org.apache.hyracks.storage.am.lsm.btree.impls; -import java.io.File; import java.io.FilenameFilter; import java.io.IOException; import java.util.ArrayList; @@ -81,7 +80,8 @@ // create transaction filter <to hide transaction files> FilenameFilter transactionFilter = getTransactionFileFilter(false); // List of valid BTree files. - cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, btreeFilter), btreeFactory, allBTreeFiles); + cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, btreeFilter), btreeFactory, allBTreeFiles, + btreeFactory.getBufferCache()); HashSet<String> btreeFilesSet = new HashSet<>(); for (ComparableFileName cmpFileName : allBTreeFiles) { int index = cmpFileName.fileName.lastIndexOf(DELIMITER); @@ -90,7 +90,7 @@ if (hasBloomFilter) { validateFiles(btreeFilesSet, allBloomFilterFiles, getCompoundFilter(transactionFilter, bloomFilterFilter), - null); + null, btreeFactory.getBufferCache()); // Sanity check. if (allBTreeFiles.size() != allBloomFilterFiles.size()) { throw HyracksDataException.create(ErrorCode.UNEQUAL_NUM_FILTERS_TREES, baseDir); @@ -148,9 +148,9 @@ && (!hasBloomFilter || (currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0 && currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0))) { // Invalid files are completely contained in last interval. - IoUtil.delete(new File(currentBTree.fullPath)); + delete(btreeFactory.getBufferCache(), currentBTree.fullPath); if (hasBloomFilter) { - IoUtil.delete(new File(currentBloomFilter.fullPath)); + delete(btreeFactory.getBufferCache(), currentBloomFilter.fullPath); } } else { // This scenario should not be possible. diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java index f61d783..23c2367 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java @@ -18,7 +18,6 @@ */ package org.apache.hyracks.storage.am.lsm.btree.impls; -import java.io.File; import java.io.FilenameFilter; import java.io.IOException; import java.nio.file.Files; @@ -91,16 +90,16 @@ FilenameFilter transactionFilefilter = getTransactionFileFilter(false); // Gather files. cleanupAndGetValidFilesInternal(getCompoundFilter(btreeFilter, transactionFilefilter), btreeFactory, - allBTreeFiles); + allBTreeFiles, btreeFactory.getBufferCache()); HashSet<String> btreeFilesSet = new HashSet<>(); for (ComparableFileName cmpFileName : allBTreeFiles) { int index = cmpFileName.fileName.lastIndexOf(DELIMITER); btreeFilesSet.add(cmpFileName.fileName.substring(0, index)); } validateFiles(btreeFilesSet, allBuddyBTreeFiles, getCompoundFilter(buddyBtreeFilter, transactionFilefilter), - buddyBtreeFactory); + buddyBtreeFactory, btreeFactory.getBufferCache()); validateFiles(btreeFilesSet, allBloomFilterFiles, getCompoundFilter(bloomFilterFilter, transactionFilefilter), - null); + null, btreeFactory.getBufferCache()); // Sanity check. if (allBTreeFiles.size() != allBuddyBTreeFiles.size() || allBTreeFiles.size() != allBloomFilterFiles.size()) { throw HyracksDataException.create(ErrorCode.UNEQUAL_NUM_FILTERS_TREES, baseDir); @@ -153,9 +152,9 @@ && currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0 && currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0) { // Invalid files are completely contained in last interval. - IoUtil.delete(new File(currentBTree.fullPath)); - IoUtil.delete(new File(currentBuddyBTree.fullPath)); - IoUtil.delete(new File(currentBloomFilter.fullPath)); + delete(treeFactory.getBufferCache(), currentBTree.fullPath); + delete(treeFactory.getBufferCache(), currentBuddyBTree.fullPath); + delete(treeFactory.getBufferCache(), currentBloomFilter.fullPath); } else { // This scenario should not be possible. throw HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir); diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java index 59a919b..42d3f56 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java @@ -19,7 +19,6 @@ package org.apache.hyracks.storage.am.lsm.common.impls; -import java.io.File; import java.io.FilenameFilter; import java.io.IOException; import java.text.Format; @@ -132,8 +131,8 @@ } protected void cleanupAndGetValidFilesInternal(FilenameFilter filter, - TreeIndexFactory<? extends ITreeIndex> treeFactory, ArrayList<ComparableFileName> allFiles) - throws HyracksDataException { + TreeIndexFactory<? extends ITreeIndex> treeFactory, ArrayList<ComparableFileName> allFiles, + IBufferCache bufferCache) throws HyracksDataException { String[] files = listDirFiles(baseDir, filter); for (String fileName : files) { FileReference fileRef = baseDir.getChild(fileName); @@ -145,7 +144,7 @@ if (idxState == TreeIndexState.VALID) { allFiles.add(new ComparableFileName(fileRef)); } else if (idxState == TreeIndexState.INVALID) { - fileRef.delete(); + bufferCache.deleteFile(fileRef); } } } @@ -169,17 +168,17 @@ } protected void validateFiles(HashSet<String> groundTruth, ArrayList<ComparableFileName> validFiles, - FilenameFilter filter, TreeIndexFactory<? extends ITreeIndex> treeFactory) throws HyracksDataException { + FilenameFilter filter, TreeIndexFactory<? extends ITreeIndex> treeFactory, IBufferCache bufferCache) + throws HyracksDataException { ArrayList<ComparableFileName> tmpAllInvListsFiles = new ArrayList<>(); - cleanupAndGetValidFilesInternal(filter, treeFactory, tmpAllInvListsFiles); + cleanupAndGetValidFilesInternal(filter, treeFactory, tmpAllInvListsFiles, bufferCache); for (ComparableFileName cmpFileName : tmpAllInvListsFiles) { int index = cmpFileName.fileName.lastIndexOf(DELIMITER); String file = cmpFileName.fileName.substring(0, index); if (groundTruth.contains(file)) { validFiles.add(cmpFileName); } else { - File invalidFile = new File(cmpFileName.fullPath); - IoUtil.delete(invalidFile); + delete(bufferCache, cmpFileName.fullPath); } } } @@ -224,7 +223,7 @@ // (1) The isValid flag is not set // (2) The file's interval is contained by some other file // Here, we only filter out (1). - cleanupAndGetValidFilesInternal(COMPONENT_FILES_FILTER, treeFactory, allFiles); + cleanupAndGetValidFilesInternal(COMPONENT_FILES_FILTER, treeFactory, allFiles, treeFactory.getBufferCache()); if (allFiles.isEmpty()) { return validFiles; @@ -252,7 +251,7 @@ // The current file is completely contained in the interval of the // last file. Thus the last file must contain at least as much information // as the current file, so delete the current file. - current.fileRef.delete(); + delete(treeFactory.getBufferCache(), current.fullPath); } else { // This scenario should not be possible since timestamps are monotonically increasing. throw HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir); @@ -387,6 +386,11 @@ } } + protected void delete(IBufferCache bufferCache, String fullPath) throws HyracksDataException { + FileReference fileRef = ioManager.resolveAbsolutePath(fullPath); + bufferCache.deleteFile(fileRef); + } + protected FilenameFilter getCompoundFilter(final FilenameFilter filter1, final FilenameFilter filter2) { return (dir, name) -> filter1.accept(dir, name) && filter2.accept(dir, name); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java index c4a2f73..2f1eb87 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java @@ -19,7 +19,6 @@ package org.apache.hyracks.storage.am.lsm.invertedindex.impls; -import java.io.File; import java.io.FilenameFilter; import java.util.ArrayList; import java.util.Collections; @@ -31,7 +30,6 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; -import org.apache.hyracks.api.util.IoUtil; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; @@ -88,7 +86,8 @@ ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<>(); // Gather files. - cleanupAndGetValidFilesInternal(deletedKeysBTreeFilter, btreeFactory, allDeletedKeysBTreeFiles); + cleanupAndGetValidFilesInternal(deletedKeysBTreeFilter, btreeFactory, allDeletedKeysBTreeFiles, + btreeFactory.getBufferCache()); HashSet<String> deletedKeysBTreeFilesSet = new HashSet<>(); for (ComparableFileName cmpFileName : allDeletedKeysBTreeFiles) { int index = cmpFileName.fileName.lastIndexOf(DELIMITER); @@ -96,9 +95,11 @@ } // TODO: do we really need to validate the inverted lists files or is validating the dict. BTrees is enough? - validateFiles(deletedKeysBTreeFilesSet, allInvListsFiles, invListFilter, null); - validateFiles(deletedKeysBTreeFilesSet, allDictBTreeFiles, dictBTreeFilter, btreeFactory); - validateFiles(deletedKeysBTreeFilesSet, allBloomFilterFiles, bloomFilterFilter, null); + validateFiles(deletedKeysBTreeFilesSet, allInvListsFiles, invListFilter, null, btreeFactory.getBufferCache()); + validateFiles(deletedKeysBTreeFilesSet, allDictBTreeFiles, dictBTreeFilter, btreeFactory, + btreeFactory.getBufferCache()); + validateFiles(deletedKeysBTreeFilesSet, allBloomFilterFiles, bloomFilterFilter, null, + btreeFactory.getBufferCache()); // Sanity check. if (allDictBTreeFiles.size() != allInvListsFiles.size() @@ -158,9 +159,9 @@ && currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0 && currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0) { // Invalid files are completely contained in last interval. - IoUtil.delete(new File(currentDeletedKeysBTree.fullPath)); - IoUtil.delete(new File(currentDictBTree.fullPath)); - IoUtil.delete(new File(currentBloomFilter.fullPath)); + delete(treeFactory.getBufferCache(), currentDeletedKeysBTree.fullPath); + delete(treeFactory.getBufferCache(), currentDictBTree.fullPath); + delete(treeFactory.getBufferCache(), currentBloomFilter.fullPath); } else { // This scenario should not be possible. throw HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir); diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java index 001228c..2512776 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java @@ -19,7 +19,6 @@ package org.apache.hyracks.storage.am.lsm.rtree.impls; -import java.io.File; import java.io.FilenameFilter; import java.io.IOException; import java.nio.file.Files; @@ -91,15 +90,17 @@ FilenameFilter transactionFilter = getTransactionFileFilter(false); // Gather files. - cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, btreeFilter), btreeFactory, allBTreeFiles); + cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, btreeFilter), btreeFactory, allBTreeFiles, + btreeFactory.getBufferCache()); HashSet<String> btreeFilesSet = new HashSet<>(); for (ComparableFileName cmpFileName : allBTreeFiles) { int index = cmpFileName.fileName.lastIndexOf(DELIMITER); btreeFilesSet.add(cmpFileName.fileName.substring(0, index)); } - validateFiles(btreeFilesSet, allRTreeFiles, getCompoundFilter(transactionFilter, rtreeFilter), rtreeFactory); - validateFiles(btreeFilesSet, allBloomFilterFiles, getCompoundFilter(transactionFilter, bloomFilterFilter), - null); + validateFiles(btreeFilesSet, allRTreeFiles, getCompoundFilter(transactionFilter, rtreeFilter), rtreeFactory, + btreeFactory.getBufferCache()); + validateFiles(btreeFilesSet, allBloomFilterFiles, getCompoundFilter(transactionFilter, bloomFilterFilter), null, + btreeFactory.getBufferCache()); // Sanity check. if (allRTreeFiles.size() != allBTreeFiles.size() || allBTreeFiles.size() != allBloomFilterFiles.size()) { @@ -155,9 +156,9 @@ && currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0 && currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0) { // Invalid files are completely contained in last interval. - IoUtil.delete(new File(currentRTree.fullPath)); - IoUtil.delete(new File(currentBTree.fullPath)); - IoUtil.delete(new File(currentBloomFilter.fullPath)); + delete(treeFactory.getBufferCache(), currentRTree.fullPath); + delete(treeFactory.getBufferCache(), currentBTree.fullPath); + delete(treeFactory.getBufferCache(), currentBloomFilter.fullPath); } else { // This scenario should not be possible. throw HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir); diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java index 673a27f..7b7e850 100644 --- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java +++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java @@ -26,6 +26,7 @@ import org.apache.hyracks.api.application.INCServiceContext; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.exceptions.HyracksException; +import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.api.io.IODeviceHandle; import org.apache.hyracks.control.nc.io.DefaultDeviceResolver; import org.apache.hyracks.control.nc.io.IOManager; @@ -88,11 +89,8 @@ public synchronized static IBufferCache getBufferCache(INCServiceContext ctx) { if (bufferCache == null) { - ICacheMemoryAllocator allocator = new HeapBufferAllocator(); - IPageReplacementStrategy prs = new ClockPageReplacementStrategy(allocator, pageSize, numPages); - IFileMapProvider fileMapProvider = getFileMapProvider(); - bufferCache = new BufferCache(ctx.getIoManager(), prs, new DelayPageCleanerPolicy(1000), - (IFileMapManager) fileMapProvider, maxOpenFiles, threadFactory); + IIOManager ioManager = ctx.getIoManager(); + return getBufferCache(ioManager); } return bufferCache; } @@ -145,4 +143,16 @@ } return resourceIdFactory; } + + public static IBufferCache getBufferCache(IIOManager ioManager) { + if (bufferCache != null) { + return bufferCache; + } + ICacheMemoryAllocator allocator = new HeapBufferAllocator(); + IPageReplacementStrategy prs = new ClockPageReplacementStrategy(allocator, pageSize, numPages); + IFileMapProvider fileMapProvider = getFileMapProvider(); + bufferCache = new BufferCache(ioManager, prs, new DelayPageCleanerPolicy(1000), + (IFileMapManager) fileMapProvider, maxOpenFiles, threadFactory); + return bufferCache; + } } diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java index 87fb11a..8ecbcc4 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java @@ -28,17 +28,19 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndex; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; +import org.apache.hyracks.storage.common.buffercache.IBufferCache; public class TestLsmIndexFileManager extends AbstractLSMIndexFileManager { - public TestLsmIndexFileManager(IIOManager ioManager, FileReference file) { - super(ioManager, file, null); + public TestLsmIndexFileManager(IIOManager ioManager, FileReference file, + TreeIndexFactory<? extends ITreeIndex> treeIndexFactory) { + super(ioManager, file, treeIndexFactory); } @Override protected void cleanupAndGetValidFilesInternal(FilenameFilter filter, - TreeIndexFactory<? extends ITreeIndex> treeFactory, ArrayList<ComparableFileName> allFiles) - throws HyracksDataException { + TreeIndexFactory<? extends ITreeIndex> treeFactory, ArrayList<ComparableFileName> allFiles, + IBufferCache bufferCache) throws HyracksDataException { String[] files = baseDir.getFile().list(filter); for (String fileName : files) { FileReference fileRef = baseDir.getChild(fileName); diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMIndexFileManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMIndexFileManagerTest.java index 8c1124f..22456e8 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMIndexFileManagerTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMIndexFileManagerTest.java @@ -38,13 +38,17 @@ import org.apache.hyracks.api.util.IoUtil; import org.apache.hyracks.control.nc.io.DefaultDeviceResolver; import org.apache.hyracks.control.nc.io.IOManager; +import org.apache.hyracks.storage.am.common.api.ITreeIndex; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.component.TestLsmIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; +import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; +import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.test.support.TestStorageManagerComponentHolder; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; public class LSMIndexFileManagerTest { private static final int DEFAULT_PAGE_SIZE = 256; @@ -54,6 +58,7 @@ protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS"); protected final static String sep = System.getProperty("file.separator"); protected IOManager ioManager; + protected IBufferCache bufferCache; protected String baseDir; protected FileReference file; @@ -61,6 +66,7 @@ public void setUp() throws HyracksDataException { TestStorageManagerComponentHolder.init(DEFAULT_PAGE_SIZE, DEFAULT_NUM_PAGES, DEFAULT_MAX_OPEN_FILES); ioManager = TestStorageManagerComponentHolder.getIOManager(); + bufferCache = TestStorageManagerComponentHolder.getBufferCache(ioManager); baseDir = ioManager.getIODevices().get(DEFAULT_IO_DEVICE_ID).getMount() + sep + "lsm_tree" + simpleDateFormat.format(new Date()) + sep; File f = new File(baseDir); @@ -75,7 +81,9 @@ } public void sortOrderTest(boolean testFlushFileName) throws InterruptedException, HyracksDataException { - ILSMIndexFileManager fileManager = new TestLsmIndexFileManager(ioManager, file); + TreeIndexFactory<? extends ITreeIndex> treeIndexFactory = Mockito.mock(TreeIndexFactory.class); + Mockito.when(treeIndexFactory.getBufferCache()).thenReturn(bufferCache); + ILSMIndexFileManager fileManager = new TestLsmIndexFileManager(ioManager, file, treeIndexFactory); LinkedList<String> fileNames = new LinkedList<>(); int numFileNames = 100; @@ -122,7 +130,9 @@ IoUtil.delete(f); } FileReference file = ioManager.resolveAbsolutePath(f.getAbsolutePath()); - ILSMIndexFileManager fileManager = new TestLsmIndexFileManager(ioManager, file); + TreeIndexFactory<? extends ITreeIndex> treeIndexFactory = Mockito.mock(TreeIndexFactory.class); + Mockito.when(treeIndexFactory.getBufferCache()).thenReturn(bufferCache); + ILSMIndexFileManager fileManager = new TestLsmIndexFileManager(ioManager, file, treeIndexFactory); fileManager.createDirs(); List<FileReference> flushFiles = new ArrayList<>(); -- To view, visit https://asterix-gerrit.ics.uci.edu/2822 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: If0f11bc222662e4b50c1b47b1dfa6b30d1463b2e Gerrit-PatchSet: 9 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: abdullah alamoudi <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Dmitry Lychagin <[email protected]> Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Luo Chen <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: Wail Alkowaileet <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
