http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java index fca010f..ead9db4 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java @@ -33,11 +33,11 @@ import java.util.Date; import java.util.HashSet; import java.util.List; -import org.apache.commons.io.FileUtils; import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; +import org.apache.hyracks.api.util.IoUtil; import org.apache.hyracks.storage.am.common.api.ITreeIndex; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame; @@ -45,49 +45,63 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.file.BufferedFileHandle; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManager { - public static final String SPLIT_STRING = "_"; - protected static final String BLOOM_FILTER_STRING = "f"; - protected static final String TRANSACTION_PREFIX = ".T"; - public enum TreeIndexState { INVALID, VERSION_MISMATCH, VALID } + /** + * Split different parts of the file name + */ + public static final String DELIMITER = "_"; + /** + * Indicates a B tree + */ + public static final String BTREE_SUFFIX = "b"; + /** + * Indicates an R tree + */ + public static final String RTREE_SUFFIX = "r"; + /** + * Indicates a bloom filter + */ + public static final String BLOOM_FILTER_SUFFIX = "f"; + /** + * Indicates a delete tree + */ + public static final String DELETE_TREE_SUFFIX = "d"; + /** + * Hides transaction components until they are either committed by removing this file or deleted along with the file + */ + public static final String TXN_PREFIX = ".T"; + + protected static final FilenameFilter fileNameFilter = (dir, name) -> !name.startsWith("."); + protected static final FilenameFilter txnFileNameFilter = (dir, name) -> name.startsWith(TXN_PREFIX); + protected static FilenameFilter bloomFilterFilter = + (dir, name) -> !name.startsWith(".") && name.endsWith(BLOOM_FILTER_SUFFIX); + protected static final FilenameFilter dummyFilter = (dir, name) -> true; + protected static final Comparator<String> cmp = new FileNameComparator(); + protected final IIOManager ioManager; - protected final IFileMapProvider fileMapProvider; // baseDir should reflect dataset name and partition name and be absolute - protected String baseDir; + protected final String baseDir; protected final Format formatter = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS"); - protected final Comparator<String> cmp = new FileNameComparator(); protected final Comparator<ComparableFileName> recencyCmp = new RecencyComparator(); protected final TreeIndexFactory<? extends ITreeIndex> treeFactory; - private String prevTimestamp = null; - public AbstractLSMIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file, + public AbstractLSMIndexFileManager(IIOManager ioManager, FileReference file, TreeIndexFactory<? extends ITreeIndex> treeFactory) { this.ioManager = ioManager; - this.baseDir = file.getFile().getAbsolutePath(); - if (!baseDir.endsWith(System.getProperty("file.separator"))) { - baseDir += System.getProperty("file.separator"); - } - this.fileMapProvider = fileMapProvider; + this.baseDir = file.getFile().getAbsolutePath().endsWith(File.separator) ? file.getFile().getAbsolutePath() + : file.getFile().getAbsolutePath() + File.separator; this.treeFactory = treeFactory; } - private static FilenameFilter fileNameFilter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return !name.startsWith("."); - } - }; - protected TreeIndexState isValidTreeIndex(ITreeIndex treeIndex) throws HyracksDataException { IBufferCache bufferCache = treeIndex.getBufferCache(); treeIndex.activate(); @@ -148,7 +162,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage if (!dir.canRead()) { throw HyracksDataException.create(ErrorCode.CANNOT_READ_FILE, path); } else if (!dir.exists()) { - throw HyracksDataException.create(ErrorCode.FILE_DOES_NOT_EXISTS, path); + throw HyracksDataException.create(ErrorCode.FILE_DOES_NOT_EXIST, path); } else if (!dir.isDirectory()) { throw HyracksDataException.create(ErrorCode.FILE_IS_NOT_DIRECTORY, path); } @@ -162,44 +176,31 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage ArrayList<ComparableFileName> tmpAllInvListsFiles = new ArrayList<>(); cleanupAndGetValidFilesInternal(filter, treeFactory, tmpAllInvListsFiles); for (ComparableFileName cmpFileName : tmpAllInvListsFiles) { - int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING); + int index = cmpFileName.fileName.lastIndexOf(DELIMITER); String file = cmpFileName.fileName.substring(0, index); if (groundTruth.contains(file)) { validFiles.add(cmpFileName); } else { File invalidFile = new File(cmpFileName.fullPath); - invalidFile.delete(); + IoUtil.delete(invalidFile); } } } @Override - public void createDirs() { + public void createDirs() throws HyracksDataException { File f = new File(baseDir); + if (f.exists()) { + throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_EXISTING_INDEX); + } f.mkdirs(); } @Override public void deleteDirs() throws HyracksDataException { - File f = new File(baseDir); - if (f.exists()) { - delete(f); - } - } - - private void delete(File f) throws HyracksDataException { - if (!FileUtils.deleteQuietly(f)) { - throw HyracksDataException.create(ErrorCode.UNIDENTIFIED_IO_ERROR_DELETING_DIR, f.getPath()); - } + IoUtil.delete(new File(baseDir)); } - protected static FilenameFilter bloomFilterFilter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return !name.startsWith(".") && name.endsWith(BLOOM_FILTER_STRING); - } - }; - protected FileReference createFlushFile(String flushFileName) throws HyracksDataException { return ioManager.resolveAbsolutePath(flushFileName); } @@ -212,17 +213,17 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException { String ts = getCurrentTimestamp(); // Begin timestamp and end timestamp are identical since it is a flush - return new LSMComponentFileReferences(createFlushFile(baseDir + ts + SPLIT_STRING + ts), null, null); + return new LSMComponentFileReferences(createFlushFile(baseDir + ts + DELIMITER + ts), null, null); } @Override public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName) throws HyracksDataException { - String[] firstTimestampRange = firstFileName.split(SPLIT_STRING); - String[] lastTimestampRange = lastFileName.split(SPLIT_STRING); + String[] firstTimestampRange = firstFileName.split(DELIMITER); + String[] lastTimestampRange = lastFileName.split(DELIMITER); // Get the range of timestamps by taking the earliest and the latest timestamps return new LSMComponentFileReferences( - createMergeFile(baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1]), null, null); + createMergeFile(baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]), null, null); } @Override @@ -292,7 +293,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage * 2. Flushed files are sorted from newest to oldest (based on the timestamp * string) */ - private class FileNameComparator implements Comparator<String> { + private static class FileNameComparator implements Comparator<String> { @Override public int compare(String a, String b) { // Consciously ignoring locale. @@ -307,7 +308,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage @Override public void recoverTransaction() throws HyracksDataException { - String[] files = listDirFiles(baseDir, transactionFileNameFilter); + String[] files = listDirFiles(baseDir, txnFileNameFilter); File dir = new File(baseDir); try { if (files.length == 0) { @@ -334,7 +335,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage this.fileRef = fileRef; this.fullPath = fileRef.getFile().getAbsolutePath(); this.fileName = fileRef.getFile().getName(); - interval = fileName.split(SPLIT_STRING); + interval = fileName.split(DELIMITER); } @Override @@ -361,7 +362,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage // This function is used to delete transaction files for aborted transactions @Override public void deleteTransactionFiles() throws HyracksDataException { - String[] files = listDirFiles(baseDir, transactionFileNameFilter); + String[] files = listDirFiles(baseDir, txnFileNameFilter); if (files.length == 0) { // Do nothing } else if (files.length > 1) { @@ -399,23 +400,9 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage return null; } - protected static FilenameFilter transactionFileNameFilter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.startsWith(".T"); - } - }; - - protected static FilenameFilter dummyFilter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return true; - } - }; - protected static FilenameFilter createTransactionFilter(String transactionFileName, final boolean inclusive) { - final String timeStamp = transactionFileName - .substring(transactionFileName.indexOf(TRANSACTION_PREFIX) + TRANSACTION_PREFIX.length()); + final String timeStamp = + transactionFileName.substring(transactionFileName.indexOf(TXN_PREFIX) + TXN_PREFIX.length()); return new FilenameFilter() { @Override public boolean accept(File dir, String name) { @@ -429,7 +416,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage } protected FilenameFilter getTransactionFileFilter(boolean inclusive) throws HyracksDataException { - String[] files = listDirFiles(baseDir, transactionFileNameFilter); + String[] files = listDirFiles(baseDir, txnFileNameFilter); if (files.length == 0) { return dummyFilter; } else {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java index 562ed5a..4eefcd8 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java @@ -26,21 +26,20 @@ import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public class BTreeFactory extends TreeIndexFactory<BTree> { - public BTreeFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider, - IPageManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory, - ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) { - super(ioManager, bufferCache, fileMapProvider, freePageManagerFactory, interiorFrameFactory, leafFrameFactory, - cmpFactories, fieldCount); + public BTreeFactory(IIOManager ioManager, IBufferCache bufferCache, IPageManagerFactory freePageManagerFactory, + ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory, + IBinaryComparatorFactory[] cmpFactories, int fieldCount) { + super(ioManager, bufferCache, freePageManagerFactory, interiorFrameFactory, leafFrameFactory, cmpFactories, + fieldCount); } @Override public BTree createIndexInstance(FileReference file) { - return new BTree(bufferCache, fileMapProvider, freePageManagerFactory.createPageManager(bufferCache), - interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, file); + return new BTree(bufferCache, freePageManagerFactory.createPageManager(bufferCache), interiorFrameFactory, + leafFrameFactory, cmpFactories, fieldCount, file); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java index 55fc92c..7a05d6d 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java @@ -25,20 +25,16 @@ import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; import org.apache.hyracks.storage.common.IIndex; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public abstract class IndexFactory<T extends IIndex> { protected final IIOManager ioManager; protected final IBufferCache bufferCache; - protected final IFileMapProvider fileMapProvider; protected final IPageManagerFactory freePageManagerFactory; - public IndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider, - IPageManagerFactory freePageManagerFactory) { + public IndexFactory(IIOManager ioManager, IBufferCache bufferCache, IPageManagerFactory freePageManagerFactory) { this.ioManager = ioManager; this.bufferCache = bufferCache; - this.fileMapProvider = fileMapProvider; this.freePageManagerFactory = freePageManagerFactory; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java index 6878fcf..83e140c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java @@ -18,6 +18,8 @@ */ package org.apache.hyracks.storage.am.lsm.common.impls; +import java.util.HashMap; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; @@ -29,6 +31,7 @@ import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue; import org.apache.hyracks.storage.common.file.IFileMapManager; +import org.apache.hyracks.util.JSONUtil; public class MultitenantVirtualBufferCache implements IVirtualBufferCache { private static final Logger LOGGER = Logger.getLogger(ExternalIndexHarness.class.getName()); @@ -42,8 +45,8 @@ public class MultitenantVirtualBufferCache implements IVirtualBufferCache { } @Override - public void createFile(FileReference fileRef) throws HyracksDataException { - vbc.createFile(fileRef); + public int createFile(FileReference fileRef) throws HyracksDataException { + return vbc.createFile(fileRef); } @Override @@ -57,8 +60,8 @@ public class MultitenantVirtualBufferCache implements IVirtualBufferCache { } @Override - public void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException { - vbc.deleteFile(fileId, flushDirtyPages); + public void deleteFile(int fileId) throws HyracksDataException { + vbc.deleteFile(fileId); } @Override @@ -91,6 +94,7 @@ public class MultitenantVirtualBufferCache implements IVirtualBufferCache { return vbc.getPageSize(); } + @Override public int getPageSizeWithHeader() { return vbc.getPageSizeWithHeader(); } @@ -133,7 +137,7 @@ public class MultitenantVirtualBufferCache implements IVirtualBufferCache { @Override public int getNumPagesOfFile(int fileId) throws HyracksDataException { - throw new UnsupportedOperationException(); + return vbc.getNumPagesOfFile(fileId); } @Override @@ -205,4 +209,28 @@ public class MultitenantVirtualBufferCache implements IVirtualBufferCache { throws HyracksDataException { vbc.resizePage(page, multiplier, extraPageBlockHelper); } + + @Override + public String toString() { + return JSONUtil.fromMap(toMap()); + } + + private Map<String, Object> toMap() { + HashMap<String, Object> map = new HashMap<>(); + map.put("class", getClass().getSimpleName()); + map.put("vbc", vbc.toString()); + map.put("openCount", openCount); + return map; + } + + @Override + public int openFile(FileReference fileRef) throws HyracksDataException { + return vbc.openFile(fileRef); + } + + @Override + public void deleteFile(FileReference file) throws HyracksDataException { + vbc.deleteFile(file); + } + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java index 2a7bcca..b6e2dd4 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java @@ -25,7 +25,6 @@ import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; import org.apache.hyracks.storage.am.common.api.ITreeIndex; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public abstract class TreeIndexFactory<T extends ITreeIndex> extends IndexFactory<T> { @@ -34,10 +33,10 @@ public abstract class TreeIndexFactory<T extends ITreeIndex> extends IndexFactor protected final IBinaryComparatorFactory[] cmpFactories; protected final int fieldCount; - public TreeIndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider, - IPageManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory, - ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) { - super(ioManager, bufferCache, fileMapProvider, freePageManagerFactory); + public TreeIndexFactory(IIOManager ioManager, IBufferCache bufferCache, IPageManagerFactory freePageManagerFactory, + ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory, + IBinaryComparatorFactory[] cmpFactories, int fieldCount) { + super(ioManager, bufferCache, freePageManagerFactory); this.interiorFrameFactory = interiorFrameFactory; this.leafFrameFactory = leafFrameFactory; this.cmpFactories = cmpFactories; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java index 27d879c..d26677e 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java @@ -21,6 +21,8 @@ package org.apache.hyracks.storage.am.lsm.common.impls; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; import java.util.logging.Level; @@ -37,7 +39,8 @@ import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue; import org.apache.hyracks.storage.common.buffercache.VirtualPage; import org.apache.hyracks.storage.common.file.BufferedFileHandle; import org.apache.hyracks.storage.common.file.IFileMapManager; -import org.apache.hyracks.storage.common.file.TransientFileMapManager; +import org.apache.hyracks.storage.common.file.FileMapManager; +import org.apache.hyracks.util.JSONUtil; public class VirtualBufferCache implements IVirtualBufferCache { private static final Logger LOGGER = Logger.getLogger(ExternalIndexHarness.class.getName()); @@ -59,7 +62,7 @@ public class VirtualBufferCache implements IVirtualBufferCache { public VirtualBufferCache(ICacheMemoryAllocator allocator, int pageSize, int numPages) { this.allocator = allocator; - this.fileMapManager = new TransientFileMapManager(); + this.fileMapManager = new FileMapManager(); this.pageSize = pageSize; this.numPages = 2 * (numPages / 2) + 1; @@ -71,12 +74,17 @@ public class VirtualBufferCache implements IVirtualBufferCache { } @Override - public void createFile(FileReference fileRef) throws HyracksDataException { + public int createFile(FileReference fileRef) throws HyracksDataException { + return fileMapManager.registerFile(fileRef); + } + + @Override + public int openFile(FileReference fileRef) throws HyracksDataException { synchronized (fileMapManager) { if (fileMapManager.isMapped(fileRef)) { - throw new HyracksDataException("File " + fileRef + " is already mapped"); + return fileMapManager.lookupFileId(fileRef); } - fileMapManager.registerFile(fileRef); + return fileMapManager.registerFile(fileRef); } } @@ -89,14 +97,16 @@ public class VirtualBufferCache implements IVirtualBufferCache { } @Override - public void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException { + public void deleteFile(FileReference fileRef) throws HyracksDataException { synchronized (fileMapManager) { - if (!fileMapManager.isMapped(fileId)) { - throw new HyracksDataException("File with id " + fileId + " is not mapped"); - } - fileMapManager.unregisterFile(fileId); + int fileId = fileMapManager.lookupFileId(fileRef); + deleteFile(fileId); } + } + @Override + public void deleteFile(int fileId) throws HyracksDataException { + fileMapManager.unregisterFile(fileId); for (int i = 0; i < buckets.length; i++) { final CacheBucket bucket = buckets[i]; bucket.bucketLock.lock(); @@ -355,12 +365,7 @@ public class VirtualBufferCache implements IVirtualBufferCache { @Override public int getNumPagesOfFile(int fileId) throws HyracksDataException { - synchronized (fileMapManager) { - if (fileMapManager.isMapped(fileId)) { - return numPages; - } - } - return 0; + return -1; } @Override @@ -424,7 +429,21 @@ public class VirtualBufferCache implements IVirtualBufferCache { @Override public void purgeHandle(int fileId) throws HyracksDataException { + deleteFile(fileId); + } + @Override + public String toString() { + return JSONUtil.fromMap(toMap()); } + private Map<String, Object> toMap() { + HashMap<String, Object> map = new HashMap<>(); + map.put("class", getClass().getSimpleName()); + map.put("allocator", allocator.toString()); + map.put("pageSize", pageSize); + map.put("numPages", numPages); + map.put("open", open); + return map; + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml index 0f4063a..b94ebee 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml @@ -16,7 +16,8 @@ ! specific language governing permissions and limitations ! under the License. !--> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <artifactId>hyracks-storage-am-lsm-invertedindex</artifactId> <parent> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java index 7258076..eceb7b4 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java @@ -39,4 +39,13 @@ public interface IInvertedIndex extends IIndex { ITypeTraits[] getTokenTypeTraits(); IBinaryComparatorFactory[] getTokenCmpFactories(); + + /** + * Purge the index files out of the buffer cache. + * Can only be called if the caller is absolutely sure the files don't contain dirty pages + * + * @throws HyracksDataException + * if the index is active + */ + void purge() throws HyracksDataException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java index 45ca106..7e2ec50 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java @@ -42,7 +42,6 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokeniz import org.apache.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexUtils; import org.apache.hyracks.storage.common.IStorageManager; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public class LSMInvertedIndexLocalResource extends LsmResource { @@ -86,25 +85,23 @@ public class LSMInvertedIndexLocalResource extends LsmResource { FileReference file = ioManager.resolve(path); List<IVirtualBufferCache> virtualBufferCaches = vbcProvider.getVirtualBufferCaches(serviceCtx, file); IBufferCache bufferCache = storageManager.getBufferCache(serviceCtx); - IFileMapProvider fileMapManager = storageManager.getFileMapProvider(serviceCtx); ILSMMergePolicy mergePolicy = mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx); ILSMIOOperationScheduler ioScheduler = ioSchedulerProvider.getIoScheduler(serviceCtx); if (isPartitioned) { - return InvertedIndexUtils.createPartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, fileMapManager, - typeTraits, cmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache, - file.getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy, - opTrackerProvider.getOperationTracker(serviceCtx), ioScheduler, - ioOpCallbackFactory.createIoOpCallback(), invertedIndexFields, filterTypeTraits, filterCmpFactories, - filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable, - metadataPageManagerFactory); - } else { - return InvertedIndexUtils.createLSMInvertedIndex(ioManager, virtualBufferCaches, fileMapManager, typeTraits, + return InvertedIndexUtils.createPartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, typeTraits, cmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache, file.getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy, opTrackerProvider.getOperationTracker(serviceCtx), ioScheduler, ioOpCallbackFactory.createIoOpCallback(), invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable, metadataPageManagerFactory); + } else { + return InvertedIndexUtils.createLSMInvertedIndex(ioManager, virtualBufferCaches, typeTraits, cmpFactories, + tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache, file.getAbsolutePath(), + bloomFilterFalsePositiveRate, mergePolicy, opTrackerProvider.getOperationTracker(serviceCtx), + ioScheduler, ioOpCallbackFactory.createIoOpCallback(), invertedIndexFields, filterTypeTraits, + filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, + invertedIndexFieldsForNonBulkLoadOps, durable, metadataPageManagerFactory); } } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java index d5edbb5..f3f2ff0 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java @@ -87,7 +87,6 @@ import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.ISearchPredicate; import org.apache.hyracks.storage.common.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex { private static final Logger LOGGER = Logger.getLogger(LSMInvertedIndex.class.getName()); @@ -110,14 +109,14 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory, BloomFilterFactory bloomFilterFactory, IComponentFilterHelper filterHelper, ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, - double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, - ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, - ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, - IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, - ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, - int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, - boolean durable) throws HyracksDataException { - super(ioManager, virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager, diskFileMapProvider, + double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager, ITypeTraits[] invListTypeTraits, + IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, + IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, + ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, + ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, int[] filterFields, + int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) + throws HyracksDataException { + super(ioManager, virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory, filterManager, filterFields, durable, filterHelper, invertedIndexFields); this.tokenizerFactory = tokenizerFactory; @@ -134,10 +133,10 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) { InMemoryInvertedIndex memInvIndex = createInMemoryInvertedIndex(virtualBufferCache, new VirtualFreePageManager(virtualBufferCache), i); - BTree deleteKeysBTree = BTreeUtils.createBTree(virtualBufferCache, - new VirtualFreePageManager(virtualBufferCache), virtualBufferCache.getFileMapProvider(), - invListTypeTraits, invListCmpFactories, BTreeLeafFrameType.REGULAR_NSM, - ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_del_" + i)); + BTree deleteKeysBTree = + BTreeUtils.createBTree(virtualBufferCache, new VirtualFreePageManager(virtualBufferCache), + invListTypeTraits, invListCmpFactories, BTreeLeafFrameType.REGULAR_NSM, + ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_del_" + i)); LSMInvertedIndexMemoryComponent mutableComponent = new LSMInvertedIndexMemoryComponent(memInvIndex, deleteKeysBTree, virtualBufferCache, i == 0 ? true : false, filterHelper == null ? null : filterHelper.createFilter()); @@ -177,8 +176,11 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException { LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c; component.getBloomFilter().deactivate(); + component.getBloomFilter().purge(); component.getInvIndex().deactivate(); + component.getInvIndex().purge(); component.getDeletedKeysBTree().deactivate(); + component.getDeletedKeysBTree().purge(); } @Override @@ -189,13 +191,6 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex component.getBloomFilter().destroy(); } - @Override - protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { - LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c; - mutableComponent.getInvIndex().destroy(); - mutableComponent.getDeletedKeysBTree().destroy(); - } - /** * The keys in the in-memory deleted-keys BTree only refer to on-disk components. * We delete documents from the in-memory inverted index by deleting its entries directly, @@ -741,4 +736,9 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getDeleteIndexFileReference(), mergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir()); } + + @Override + public void purge() throws HyracksDataException { + throw new UnsupportedOperationException(); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java index aeb1e16..9254c45 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java @@ -34,7 +34,6 @@ import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManage import org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNameMapper; -import org.apache.hyracks.storage.common.file.IFileMapProvider; // TODO: Refactor for better code sharing with other file managers. public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager implements IInvertedIndexFileNameMapper { @@ -66,33 +65,32 @@ public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager imp } }; - public LSMInvertedIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file, - BTreeFactory btreeFactory) { - super(ioManager, fileMapProvider, file, null); + public LSMInvertedIndexFileManager(IIOManager ioManager, FileReference file, BTreeFactory btreeFactory) { + super(ioManager, file, null); this.btreeFactory = btreeFactory; } @Override public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException { String ts = getCurrentTimestamp(); - String baseName = baseDir + ts + SPLIT_STRING + ts; + String baseName = baseDir + ts + DELIMITER + ts; // Begin timestamp and end timestamp are identical since it is a flush - return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX), - createFlushFile(baseName + SPLIT_STRING + DELETED_KEYS_BTREE_SUFFIX), - createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING)); + return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + DICT_BTREE_SUFFIX), + createFlushFile(baseName + DELIMITER + DELETED_KEYS_BTREE_SUFFIX), + createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); } @Override public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName) throws HyracksDataException { - String[] firstTimestampRange = firstFileName.split(SPLIT_STRING); - String[] lastTimestampRange = lastFileName.split(SPLIT_STRING); + String[] firstTimestampRange = firstFileName.split(DELIMITER); + String[] lastTimestampRange = lastFileName.split(DELIMITER); - String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1]; + String baseName = baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]; // Get the range of timestamps by taking the earliest and the latest timestamps - return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX), - createMergeFile(baseName + SPLIT_STRING + DELETED_KEYS_BTREE_SUFFIX), - createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING)); + return new LSMComponentFileReferences(createMergeFile(baseName + DELIMITER + DICT_BTREE_SUFFIX), + createMergeFile(baseName + DELIMITER + DELETED_KEYS_BTREE_SUFFIX), + createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); } @Override @@ -107,7 +105,7 @@ public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager imp cleanupAndGetValidFilesInternal(deletedKeysBTreeFilter, btreeFactory, allDeletedKeysBTreeFiles); HashSet<String> deletedKeysBTreeFilesSet = new HashSet<>(); for (ComparableFileName cmpFileName : allDeletedKeysBTreeFiles) { - int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING); + int index = cmpFileName.fileName.lastIndexOf(DELIMITER); deletedKeysBTreeFilesSet.add(cmpFileName.fileName.substring(0, index)); } @@ -210,8 +208,8 @@ public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager imp @Override public String getInvListsFilePath(String dictBTreeFilePath) { - int index = dictBTreeFilePath.lastIndexOf(SPLIT_STRING); + int index = dictBTreeFilePath.lastIndexOf(DELIMITER); String file = dictBTreeFilePath.substring(0, index); - return file + SPLIT_STRING + INVLISTS_SUFFIX; + return file + DELIMITER + INVLISTS_SUFFIX; } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java index 8863427..d154356 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java @@ -41,7 +41,6 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInverted import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexUtils; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public class PartitionedLSMInvertedIndex extends LSMInvertedIndex { @@ -49,17 +48,17 @@ public class PartitionedLSMInvertedIndex extends LSMInvertedIndex { OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory, BloomFilterFactory bloomFilterFactory, IComponentFilterHelper filterHelper, ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, - double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, - ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, - ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, - IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, - ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, - int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, - boolean durable) throws HyracksDataException { + double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager, ITypeTraits[] invListTypeTraits, + IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, + IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, + ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, + ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, int[] filterFields, + int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) + throws HyracksDataException { super(ioManager, virtualBufferCaches, diskInvIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, fileManager, - diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, - tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, + invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, + mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java index 4b673e5..d686634 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java @@ -31,7 +31,6 @@ import org.apache.hyracks.storage.am.btree.util.BTreeUtils; import org.apache.hyracks.storage.am.common.api.IIndexOperationContext; import org.apache.hyracks.storage.am.common.api.IPageManager; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; -import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor; import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory; @@ -74,9 +73,8 @@ public class InMemoryInvertedIndex implements IInvertedIndex { btreeTypeTraits[tokenTypeTraits.length + i] = invListTypeTraits[i]; btreeCmpFactories[tokenTypeTraits.length + i] = invListCmpFactories[i]; } - this.btree = BTreeUtils.createBTree(virtualBufferCache, virtualFreePageManager, - ((IVirtualBufferCache) virtualBufferCache).getFileMapProvider(), btreeTypeTraits, btreeCmpFactories, - BTreeLeafFrameType.REGULAR_NSM, btreeFileRef); + this.btree = BTreeUtils.createBTree(virtualBufferCache, virtualFreePageManager, btreeTypeTraits, + btreeCmpFactories, BTreeLeafFrameType.REGULAR_NSM, btreeFileRef); } @Override @@ -219,4 +217,9 @@ public class InMemoryInvertedIndex implements IInvertedIndex { public int getNumOfFilterFields() { return 0; } + + @Override + public void purge() throws HyracksDataException { + btree.purge(); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java index 277eb64..25e4201 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java @@ -26,6 +26,7 @@ import java.nio.ByteBuffer; import org.apache.hyracks.api.context.IHyracksCommonContext; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ITypeTraits; +import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; @@ -60,7 +61,6 @@ import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue; import org.apache.hyracks.storage.common.file.BufferedFileHandle; -import org.apache.hyracks.storage.common.file.IFileMapProvider; /** * An inverted index consists of two files: 1. a file storing (paginated) @@ -94,7 +94,6 @@ public class OnDiskInvertedIndex implements IInvertedIndex { protected BTree btree; protected int rootPageId = 0; protected IBufferCache bufferCache; - protected IFileMapProvider fileMapProvider; protected int fileId = -1; protected final ITypeTraits[] invListTypeTraits; protected final IBinaryComparatorFactory[] invListCmpFactories; @@ -109,21 +108,18 @@ public class OnDiskInvertedIndex implements IInvertedIndex { protected boolean isOpen = false; protected boolean wasOpen = false; - public OnDiskInvertedIndex(IBufferCache bufferCache, IFileMapProvider fileMapProvider, - IInvertedListBuilder invListBuilder, ITypeTraits[] invListTypeTraits, - IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, - IBinaryComparatorFactory[] tokenCmpFactories, FileReference btreeFile, FileReference invListsFile, - IPageManagerFactory pageManagerFactory) throws HyracksDataException { + public OnDiskInvertedIndex(IBufferCache bufferCache, IInvertedListBuilder invListBuilder, + ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, + ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference btreeFile, + FileReference invListsFile, IPageManagerFactory pageManagerFactory) throws HyracksDataException { this.bufferCache = bufferCache; - this.fileMapProvider = fileMapProvider; this.invListBuilder = invListBuilder; this.invListTypeTraits = invListTypeTraits; this.invListCmpFactories = invListCmpFactories; this.tokenTypeTraits = tokenTypeTraits; this.tokenCmpFactories = tokenCmpFactories; - this.btree = BTreeUtils.createBTree(bufferCache, fileMapProvider, getBTreeTypeTraits(tokenTypeTraits), - tokenCmpFactories, BTreeLeafFrameType.REGULAR_NSM, btreeFile, - pageManagerFactory.createPageManager(bufferCache)); + this.btree = BTreeUtils.createBTree(bufferCache, getBTreeTypeTraits(tokenTypeTraits), tokenCmpFactories, + BTreeLeafFrameType.REGULAR_NSM, btreeFile, pageManagerFactory.createPageManager(bufferCache)); this.numTokenFields = btree.getComparatorFactories().length; this.numInvListKeys = invListCmpFactories.length; this.invListsFile = invListsFile; @@ -139,25 +135,7 @@ public class OnDiskInvertedIndex implements IInvertedIndex { throw new HyracksDataException("Failed to create since index is already open."); } btree.create(); - boolean fileIsMapped = false; - synchronized (fileMapProvider) { - fileIsMapped = fileMapProvider.isMapped(invListsFile); - if (!fileIsMapped) { - bufferCache.createFile(invListsFile); - } - fileId = fileMapProvider.lookupFileId(invListsFile); - try { - // Also creates the file if it doesn't exist yet. - bufferCache.openFile(fileId); - } catch (HyracksDataException e) { - // Revert state of buffer cache since file failed to open. - if (!fileIsMapped) { - bufferCache.deleteFile(fileId, false); - } - throw e; - } - } - bufferCache.closeFile(fileId); + fileId = bufferCache.createFile(invListsFile); } @Override @@ -166,25 +144,11 @@ public class OnDiskInvertedIndex implements IInvertedIndex { throw new HyracksDataException("Failed to activate the index since it is already activated."); } btree.activate(); - boolean fileIsMapped = false; - synchronized (fileMapProvider) { - fileIsMapped = fileMapProvider.isMapped(invListsFile); - if (!fileIsMapped) { - bufferCache.createFile(invListsFile); - } - fileId = fileMapProvider.lookupFileId(invListsFile); - try { - // Also creates the file if it doesn't exist yet. - bufferCache.openFile(fileId); - } catch (HyracksDataException e) { - // Revert state of buffer cache since file failed to open. - if (!fileIsMapped) { - bufferCache.deleteFile(fileId, false); - } - throw e; - } + if (fileId >= 0) { + bufferCache.openFile(fileId); + } else { + fileId = bufferCache.openFile(invListsFile); } - isOpen = true; wasOpen = true; } @@ -194,10 +158,8 @@ public class OnDiskInvertedIndex implements IInvertedIndex { if (!isOpen && wasOpen) { throw new HyracksDataException("Failed to deactivate the index since it is already deactivated."); } - btree.deactivate(); bufferCache.closeFile(fileId); - isOpen = false; } @@ -206,15 +168,8 @@ public class OnDiskInvertedIndex implements IInvertedIndex { if (isOpen) { throw new HyracksDataException("Failed to destroy since index is already open."); } - btree.destroy(); - invListsFile.delete(); - if (fileId == -1) { - return; - } - - bufferCache.deleteFile(fileId, false); - fileId = -1; + bufferCache.deleteFile(invListsFile); } @Override @@ -224,27 +179,9 @@ public class OnDiskInvertedIndex implements IInvertedIndex { } btree.clear(); bufferCache.closeFile(fileId); - bufferCache.deleteFile(fileId, false); - invListsFile.getFile().delete(); - - boolean fileIsMapped = false; - synchronized (fileMapProvider) { - fileIsMapped = fileMapProvider.isMapped(invListsFile); - if (!fileIsMapped) { - bufferCache.createFile(invListsFile); - } - fileId = fileMapProvider.lookupFileId(invListsFile); - try { - // Also creates the file if it doesn't exist yet. - bufferCache.openFile(fileId); - } catch (HyracksDataException e) { - // Revert state of buffer cache since file failed to open. - if (!fileIsMapped) { - bufferCache.deleteFile(fileId, false); - } - throw e; - } - } + bufferCache.deleteFile(fileId); + fileId = bufferCache.createFile(invListsFile); + bufferCache.openFile(fileId); } @Override @@ -697,4 +634,14 @@ public class OnDiskInvertedIndex implements IInvertedIndex { public int getNumOfFilterFields() { return 0; } + + @Override + public synchronized void purge() throws HyracksDataException { + if (isOpen) { + throw HyracksDataException.create(ErrorCode.CANNOT_PURGE_ACTIVE_INDEX); + } + btree.purge(); + bufferCache.purgeHandle(fileId); + fileId = -1; + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java index dacef3c..70fcd03 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java @@ -30,7 +30,6 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNam import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilderFactory; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public class OnDiskInvertedIndexFactory extends IndexFactory<IInvertedIndex> { @@ -41,12 +40,12 @@ public class OnDiskInvertedIndexFactory extends IndexFactory<IInvertedIndex> { protected final IBinaryComparatorFactory[] tokenCmpFactories; protected final IInvertedIndexFileNameMapper fileNameMapper; - public OnDiskInvertedIndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider, + public OnDiskInvertedIndexFactory(IIOManager ioManager, IBufferCache bufferCache, IInvertedListBuilderFactory invListBuilderFactory, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, IInvertedIndexFileNameMapper fileNameMapper, IPageManagerFactory pageManagerFactory) { - super(ioManager, bufferCache, fileMapProvider, pageManagerFactory); + super(ioManager, bufferCache, pageManagerFactory); this.invListBuilderFactory = invListBuilderFactory; this.invListTypeTraits = invListTypeTraits; this.invListCmpFactories = invListCmpFactories; @@ -60,8 +59,7 @@ public class OnDiskInvertedIndexFactory extends IndexFactory<IInvertedIndex> { String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getAbsolutePath()); FileReference invListsFile = ioManager.resolveAbsolutePath(invListsFilePath); IInvertedListBuilder invListBuilder = invListBuilderFactory.create(); - return new OnDiskInvertedIndex(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits, - invListCmpFactories, tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile, - freePageManagerFactory); + return new OnDiskInvertedIndex(bufferCache, invListBuilder, invListTypeTraits, invListCmpFactories, + tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile, freePageManagerFactory); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java index 452d59b..0c0110e 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java @@ -39,19 +39,17 @@ import org.apache.hyracks.storage.common.IIndexAccessor; import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public class PartitionedOnDiskInvertedIndex extends OnDiskInvertedIndex implements IPartitionedInvertedIndex { protected final int PARTITIONING_NUM_TOKENS_FIELD = 1; - public PartitionedOnDiskInvertedIndex(IBufferCache bufferCache, IFileMapProvider fileMapProvider, - IInvertedListBuilder invListBuilder, ITypeTraits[] invListTypeTraits, - IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, - IBinaryComparatorFactory[] tokenCmpFactories, FileReference btreeFile, FileReference invListsFile, - IPageManagerFactory pageManagerFactory) throws HyracksDataException { - super(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits, invListCmpFactories, tokenTypeTraits, - tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory); + public PartitionedOnDiskInvertedIndex(IBufferCache bufferCache, IInvertedListBuilder invListBuilder, + ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, + ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference btreeFile, + FileReference invListsFile, IPageManagerFactory pageManagerFactory) throws HyracksDataException { + super(bufferCache, invListBuilder, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, + btreeFile, invListsFile, pageManagerFactory); } public class PartitionedOnDiskInvertedIndexAccessor extends OnDiskInvertedIndexAccessor { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java index ebc0152..5b2888a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java @@ -29,17 +29,16 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNam import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilderFactory; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public class PartitionedOnDiskInvertedIndexFactory extends OnDiskInvertedIndexFactory { public PartitionedOnDiskInvertedIndexFactory(IIOManager ioManager, IBufferCache bufferCache, - IFileMapProvider fileMapProvider, IInvertedListBuilderFactory invListBuilderFactory, - ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, - ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, - IInvertedIndexFileNameMapper fileNameMapper, IPageManagerFactory pageManagerFactory) { - super(ioManager, bufferCache, fileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, - tokenTypeTraits, tokenCmpFactories, fileNameMapper, pageManagerFactory); + IInvertedListBuilderFactory invListBuilderFactory, ITypeTraits[] invListTypeTraits, + IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, + IBinaryComparatorFactory[] tokenCmpFactories, IInvertedIndexFileNameMapper fileNameMapper, + IPageManagerFactory pageManagerFactory) { + super(ioManager, bufferCache, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits, + tokenCmpFactories, fileNameMapper, pageManagerFactory); } @Override @@ -47,8 +46,7 @@ public class PartitionedOnDiskInvertedIndexFactory extends OnDiskInvertedIndexFa String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getAbsolutePath()); FileReference invListsFile = ioManager.resolveAbsolutePath(invListsFilePath); IInvertedListBuilder invListBuilder = invListBuilderFactory.create(); - return new PartitionedOnDiskInvertedIndex(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits, - invListCmpFactories, tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile, - freePageManagerFactory); + return new PartitionedOnDiskInvertedIndex(bufferCache, invListBuilder, invListTypeTraits, invListCmpFactories, + tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile, freePageManagerFactory); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java index 8a36f61..540e100 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java @@ -59,7 +59,6 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.PartitionedOnDiskI import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.PartitionedOnDiskInvertedIndexFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public class InvertedIndexUtils { @@ -82,25 +81,23 @@ public class InvertedIndexUtils { } public static OnDiskInvertedIndex createOnDiskInvertedIndex(IIOManager ioManager, IBufferCache bufferCache, - IFileMapProvider fileMapProvider, ITypeTraits[] invListTypeTraits, - IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, - IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile, + ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, + ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile, IPageManagerFactory pageManagerFactory) throws HyracksDataException { IInvertedListBuilder builder = new FixedSizeElementInvertedListBuilder(invListTypeTraits); FileReference btreeFile = getBTreeFile(ioManager, invListsFile); - return new OnDiskInvertedIndex(bufferCache, fileMapProvider, builder, invListTypeTraits, invListCmpFactories, - tokenTypeTraits, tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory); + return new OnDiskInvertedIndex(bufferCache, builder, invListTypeTraits, invListCmpFactories, tokenTypeTraits, + tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory); } public static PartitionedOnDiskInvertedIndex createPartitionedOnDiskInvertedIndex(IIOManager ioManager, - IBufferCache bufferCache, IFileMapProvider fileMapProvider, ITypeTraits[] invListTypeTraits, - IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, - IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile, + IBufferCache bufferCache, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, + ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile, IPageManagerFactory pageManagerFactory) throws HyracksDataException { IInvertedListBuilder builder = new FixedSizeElementInvertedListBuilder(invListTypeTraits); FileReference btreeFile = getBTreeFile(ioManager, invListsFile); - return new PartitionedOnDiskInvertedIndex(bufferCache, fileMapProvider, builder, invListTypeTraits, - invListCmpFactories, tokenTypeTraits, tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory); + return new PartitionedOnDiskInvertedIndex(bufferCache, builder, invListTypeTraits, invListCmpFactories, + tokenTypeTraits, tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory); } public static FileReference getBTreeFile(IIOManager ioManager, FileReference invListsFile) @@ -108,49 +105,46 @@ public class InvertedIndexUtils { return ioManager.resolveAbsolutePath(invListsFile.getFile().getPath() + "_btree"); } - public static BTreeFactory createDeletedKeysBTreeFactory(IIOManager ioManager, IFileMapProvider diskFileMapProvider, - ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, - IBufferCache diskBufferCache, IPageManagerFactory freePageManagerFactory) throws HyracksDataException { + public static BTreeFactory createDeletedKeysBTreeFactory(IIOManager ioManager, ITypeTraits[] invListTypeTraits, + IBinaryComparatorFactory[] invListCmpFactories, IBufferCache diskBufferCache, + IPageManagerFactory freePageManagerFactory) throws HyracksDataException { TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(invListTypeTraits); ITreeIndexFrameFactory leafFrameFactory = BTreeUtils.getLeafFrameFactory(tupleWriterFactory, BTreeLeafFrameType.REGULAR_NSM); ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory); - BTreeFactory deletedKeysBTreeFactory = - new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, - interiorFrameFactory, leafFrameFactory, invListCmpFactories, invListCmpFactories.length); - return deletedKeysBTreeFactory; + return new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, interiorFrameFactory, + leafFrameFactory, invListCmpFactories, invListCmpFactories.length); } public static LSMInvertedIndex createLSMInvertedIndex(IIOManager ioManager, - List<IVirtualBufferCache> virtualBufferCaches, IFileMapProvider diskFileMapProvider, - ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, - ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, - IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String absoluteOnDiskDir, - double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, - ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, - ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, - int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, - IMetadataPageManagerFactory pageManagerFactory) throws HyracksDataException { + List<IVirtualBufferCache> virtualBufferCaches, ITypeTraits[] invListTypeTraits, + IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, + IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, + IBufferCache diskBufferCache, String absoluteOnDiskDir, double bloomFilterFalsePositiveRate, + ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, + ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, + IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, + int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, IMetadataPageManagerFactory pageManagerFactory) + throws HyracksDataException { - BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider, - invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory); + BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, invListTypeTraits, + invListCmpFactories, diskBufferCache, pageManagerFactory); int[] bloomFilterKeyFields = new int[invListCmpFactories.length]; for (int i = 0; i < invListCmpFactories.length; i++) { bloomFilterKeyFields[i] = i; } - BloomFilterFactory bloomFilterFactory = - new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields); + BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields); FileReference onDiskDirFileRef = ioManager.resolveAbsolutePath(absoluteOnDiskDir); - LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider, - onDiskDirFileRef, deletedKeysBTreeFactory); + LSMInvertedIndexFileManager fileManager = + new LSMInvertedIndexFileManager(ioManager, onDiskDirFileRef, deletedKeysBTreeFactory); IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory(invListTypeTraits); - OnDiskInvertedIndexFactory invIndexFactory = new OnDiskInvertedIndexFactory(ioManager, diskBufferCache, - diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits, - tokenCmpFactories, fileManager, pageManagerFactory); + OnDiskInvertedIndexFactory invIndexFactory = + new OnDiskInvertedIndexFactory(ioManager, diskBufferCache, invListBuilderFactory, invListTypeTraits, + invListCmpFactories, tokenTypeTraits, tokenCmpFactories, fileManager, pageManagerFactory); ComponentFilterHelper filterHelper = null; LSMComponentFilterFrameFactory filterFrameFactory = null; @@ -163,42 +157,40 @@ public class InvertedIndexUtils { } return new LSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, - fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, - tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, - invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, - durable); + fileManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, + tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, + filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable); } public static PartitionedLSMInvertedIndex createPartitionedLSMInvertedIndex(IIOManager ioManager, - List<IVirtualBufferCache> virtualBufferCaches, IFileMapProvider diskFileMapProvider, - ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, - ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, - IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String absoluteOnDiskDir, - double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, - ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, - ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, - int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, - IPageManagerFactory pageManagerFactory) throws HyracksDataException { + List<IVirtualBufferCache> virtualBufferCaches, ITypeTraits[] invListTypeTraits, + IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, + IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, + IBufferCache diskBufferCache, String absoluteOnDiskDir, double bloomFilterFalsePositiveRate, + ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, + ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, + IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, + int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, IPageManagerFactory pageManagerFactory) + throws HyracksDataException { - BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider, - invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory); + BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, invListTypeTraits, + invListCmpFactories, diskBufferCache, pageManagerFactory); int[] bloomFilterKeyFields = new int[invListCmpFactories.length]; for (int i = 0; i < invListCmpFactories.length; i++) { bloomFilterKeyFields[i] = i; } - BloomFilterFactory bloomFilterFactory = - new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields); + BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields); FileReference onDiskDirFileRef = ioManager.resolveAbsolutePath(absoluteOnDiskDir); - LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider, - onDiskDirFileRef, deletedKeysBTreeFactory); + LSMInvertedIndexFileManager fileManager = + new LSMInvertedIndexFileManager(ioManager, onDiskDirFileRef, deletedKeysBTreeFactory); IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory(invListTypeTraits); PartitionedOnDiskInvertedIndexFactory invIndexFactory = new PartitionedOnDiskInvertedIndexFactory(ioManager, - diskBufferCache, diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, - tokenTypeTraits, tokenCmpFactories, fileManager, pageManagerFactory); + diskBufferCache, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits, + tokenCmpFactories, fileManager, pageManagerFactory); ComponentFilterHelper filterHelper = null; LSMComponentFilterFrameFactory filterFrameFactory = null; @@ -211,9 +203,8 @@ public class InvertedIndexUtils { } return new PartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, - fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, - tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, - invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, - durable); + fileManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, + tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, + filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable); } }
