http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java index ead9db4..fca010f 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java @@ -33,11 +33,11 @@ import java.util.Date; import java.util.HashSet; import java.util.List; +import org.apache.commons.io.FileUtils; import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; -import org.apache.hyracks.api.util.IoUtil; import org.apache.hyracks.storage.am.common.api.ITreeIndex; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame; @@ -45,63 +45,49 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.file.BufferedFileHandle; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManager { + public static final String SPLIT_STRING = "_"; + protected static final String BLOOM_FILTER_STRING = "f"; + protected static final String TRANSACTION_PREFIX = ".T"; + public enum TreeIndexState { INVALID, VERSION_MISMATCH, VALID } - /** - * Split different parts of the file name - */ - public static final String DELIMITER = "_"; - /** - * Indicates a B tree - */ - public static final String BTREE_SUFFIX = "b"; - /** - * Indicates an R tree - */ - public static final String RTREE_SUFFIX = "r"; - /** - * Indicates a bloom filter - */ - public static final String BLOOM_FILTER_SUFFIX = "f"; - /** - * Indicates a delete tree - */ - public static final String DELETE_TREE_SUFFIX = "d"; - /** - * Hides transaction components until they are either committed by removing this file or deleted along with the file - */ - public static final String TXN_PREFIX = ".T"; - - protected static final FilenameFilter fileNameFilter = (dir, name) -> !name.startsWith("."); - protected static final FilenameFilter txnFileNameFilter = (dir, name) -> name.startsWith(TXN_PREFIX); - protected static FilenameFilter bloomFilterFilter = - (dir, name) -> !name.startsWith(".") && name.endsWith(BLOOM_FILTER_SUFFIX); - protected static final FilenameFilter dummyFilter = (dir, name) -> true; - protected static final Comparator<String> cmp = new FileNameComparator(); - protected final IIOManager ioManager; + protected final IFileMapProvider fileMapProvider; // baseDir should reflect dataset name and partition name and be absolute - protected final String baseDir; + protected String baseDir; protected final Format formatter = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS"); + protected final Comparator<String> cmp = new FileNameComparator(); protected final Comparator<ComparableFileName> recencyCmp = new RecencyComparator(); protected final TreeIndexFactory<? extends ITreeIndex> treeFactory; + private String prevTimestamp = null; - public AbstractLSMIndexFileManager(IIOManager ioManager, FileReference file, + public AbstractLSMIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file, TreeIndexFactory<? extends ITreeIndex> treeFactory) { this.ioManager = ioManager; - this.baseDir = file.getFile().getAbsolutePath().endsWith(File.separator) ? file.getFile().getAbsolutePath() - : file.getFile().getAbsolutePath() + File.separator; + this.baseDir = file.getFile().getAbsolutePath(); + if (!baseDir.endsWith(System.getProperty("file.separator"))) { + baseDir += System.getProperty("file.separator"); + } + this.fileMapProvider = fileMapProvider; this.treeFactory = treeFactory; } + private static FilenameFilter fileNameFilter = new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return !name.startsWith("."); + } + }; + protected TreeIndexState isValidTreeIndex(ITreeIndex treeIndex) throws HyracksDataException { IBufferCache bufferCache = treeIndex.getBufferCache(); treeIndex.activate(); @@ -162,7 +148,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage if (!dir.canRead()) { throw HyracksDataException.create(ErrorCode.CANNOT_READ_FILE, path); } else if (!dir.exists()) { - throw HyracksDataException.create(ErrorCode.FILE_DOES_NOT_EXIST, path); + throw HyracksDataException.create(ErrorCode.FILE_DOES_NOT_EXISTS, path); } else if (!dir.isDirectory()) { throw HyracksDataException.create(ErrorCode.FILE_IS_NOT_DIRECTORY, path); } @@ -176,31 +162,44 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage ArrayList<ComparableFileName> tmpAllInvListsFiles = new ArrayList<>(); cleanupAndGetValidFilesInternal(filter, treeFactory, tmpAllInvListsFiles); for (ComparableFileName cmpFileName : tmpAllInvListsFiles) { - int index = cmpFileName.fileName.lastIndexOf(DELIMITER); + int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING); String file = cmpFileName.fileName.substring(0, index); if (groundTruth.contains(file)) { validFiles.add(cmpFileName); } else { File invalidFile = new File(cmpFileName.fullPath); - IoUtil.delete(invalidFile); + invalidFile.delete(); } } } @Override - public void createDirs() throws HyracksDataException { + public void createDirs() { File f = new File(baseDir); - if (f.exists()) { - throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_EXISTING_INDEX); - } f.mkdirs(); } @Override public void deleteDirs() throws HyracksDataException { - IoUtil.delete(new File(baseDir)); + File f = new File(baseDir); + if (f.exists()) { + delete(f); + } + } + + private void delete(File f) throws HyracksDataException { + if (!FileUtils.deleteQuietly(f)) { + throw HyracksDataException.create(ErrorCode.UNIDENTIFIED_IO_ERROR_DELETING_DIR, f.getPath()); + } } + protected static FilenameFilter bloomFilterFilter = new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return !name.startsWith(".") && name.endsWith(BLOOM_FILTER_STRING); + } + }; + protected FileReference createFlushFile(String flushFileName) throws HyracksDataException { return ioManager.resolveAbsolutePath(flushFileName); } @@ -213,17 +212,17 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException { String ts = getCurrentTimestamp(); // Begin timestamp and end timestamp are identical since it is a flush - return new LSMComponentFileReferences(createFlushFile(baseDir + ts + DELIMITER + ts), null, null); + return new LSMComponentFileReferences(createFlushFile(baseDir + ts + SPLIT_STRING + ts), null, null); } @Override public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName) throws HyracksDataException { - String[] firstTimestampRange = firstFileName.split(DELIMITER); - String[] lastTimestampRange = lastFileName.split(DELIMITER); + String[] firstTimestampRange = firstFileName.split(SPLIT_STRING); + String[] lastTimestampRange = lastFileName.split(SPLIT_STRING); // Get the range of timestamps by taking the earliest and the latest timestamps return new LSMComponentFileReferences( - createMergeFile(baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]), null, null); + createMergeFile(baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1]), null, null); } @Override @@ -293,7 +292,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage * 2. Flushed files are sorted from newest to oldest (based on the timestamp * string) */ - private static class FileNameComparator implements Comparator<String> { + private class FileNameComparator implements Comparator<String> { @Override public int compare(String a, String b) { // Consciously ignoring locale. @@ -308,7 +307,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage @Override public void recoverTransaction() throws HyracksDataException { - String[] files = listDirFiles(baseDir, txnFileNameFilter); + String[] files = listDirFiles(baseDir, transactionFileNameFilter); File dir = new File(baseDir); try { if (files.length == 0) { @@ -335,7 +334,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage this.fileRef = fileRef; this.fullPath = fileRef.getFile().getAbsolutePath(); this.fileName = fileRef.getFile().getName(); - interval = fileName.split(DELIMITER); + interval = fileName.split(SPLIT_STRING); } @Override @@ -362,7 +361,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage // This function is used to delete transaction files for aborted transactions @Override public void deleteTransactionFiles() throws HyracksDataException { - String[] files = listDirFiles(baseDir, txnFileNameFilter); + String[] files = listDirFiles(baseDir, transactionFileNameFilter); if (files.length == 0) { // Do nothing } else if (files.length > 1) { @@ -400,9 +399,23 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage return null; } + protected static FilenameFilter transactionFileNameFilter = new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.startsWith(".T"); + } + }; + + protected static FilenameFilter dummyFilter = new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return true; + } + }; + protected static FilenameFilter createTransactionFilter(String transactionFileName, final boolean inclusive) { - final String timeStamp = - transactionFileName.substring(transactionFileName.indexOf(TXN_PREFIX) + TXN_PREFIX.length()); + final String timeStamp = transactionFileName + .substring(transactionFileName.indexOf(TRANSACTION_PREFIX) + TRANSACTION_PREFIX.length()); return new FilenameFilter() { @Override public boolean accept(File dir, String name) { @@ -416,7 +429,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage } protected FilenameFilter getTransactionFileFilter(boolean inclusive) throws HyracksDataException { - String[] files = listDirFiles(baseDir, txnFileNameFilter); + String[] files = listDirFiles(baseDir, transactionFileNameFilter); if (files.length == 0) { return dummyFilter; } else {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java index 4eefcd8..562ed5a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java @@ -26,20 +26,21 @@ import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public class BTreeFactory extends TreeIndexFactory<BTree> { - public BTreeFactory(IIOManager ioManager, IBufferCache bufferCache, IPageManagerFactory freePageManagerFactory, - ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory, - IBinaryComparatorFactory[] cmpFactories, int fieldCount) { - super(ioManager, bufferCache, freePageManagerFactory, interiorFrameFactory, leafFrameFactory, cmpFactories, - fieldCount); + public BTreeFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider, + IPageManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory, + ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) { + super(ioManager, bufferCache, fileMapProvider, freePageManagerFactory, interiorFrameFactory, leafFrameFactory, + cmpFactories, fieldCount); } @Override public BTree createIndexInstance(FileReference file) { - return new BTree(bufferCache, freePageManagerFactory.createPageManager(bufferCache), interiorFrameFactory, - leafFrameFactory, cmpFactories, fieldCount, file); + return new BTree(bufferCache, fileMapProvider, freePageManagerFactory.createPageManager(bufferCache), + interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, file); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java index 7a05d6d..55fc92c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java @@ -25,16 +25,20 @@ import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; import org.apache.hyracks.storage.common.IIndex; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public abstract class IndexFactory<T extends IIndex> { protected final IIOManager ioManager; protected final IBufferCache bufferCache; + protected final IFileMapProvider fileMapProvider; protected final IPageManagerFactory freePageManagerFactory; - public IndexFactory(IIOManager ioManager, IBufferCache bufferCache, IPageManagerFactory freePageManagerFactory) { + public IndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider, + IPageManagerFactory freePageManagerFactory) { this.ioManager = ioManager; this.bufferCache = bufferCache; + this.fileMapProvider = fileMapProvider; this.freePageManagerFactory = freePageManagerFactory; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java index 83e140c..6878fcf 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java @@ -18,8 +18,6 @@ */ package org.apache.hyracks.storage.am.lsm.common.impls; -import java.util.HashMap; -import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; @@ -31,7 +29,6 @@ import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue; import org.apache.hyracks.storage.common.file.IFileMapManager; -import org.apache.hyracks.util.JSONUtil; public class MultitenantVirtualBufferCache implements IVirtualBufferCache { private static final Logger LOGGER = Logger.getLogger(ExternalIndexHarness.class.getName()); @@ -45,8 +42,8 @@ public class MultitenantVirtualBufferCache implements IVirtualBufferCache { } @Override - public int createFile(FileReference fileRef) throws HyracksDataException { - return vbc.createFile(fileRef); + public void createFile(FileReference fileRef) throws HyracksDataException { + vbc.createFile(fileRef); } @Override @@ -60,8 +57,8 @@ public class MultitenantVirtualBufferCache implements IVirtualBufferCache { } @Override - public void deleteFile(int fileId) throws HyracksDataException { - vbc.deleteFile(fileId); + public void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException { + vbc.deleteFile(fileId, flushDirtyPages); } @Override @@ -94,7 +91,6 @@ public class MultitenantVirtualBufferCache implements IVirtualBufferCache { return vbc.getPageSize(); } - @Override public int getPageSizeWithHeader() { return vbc.getPageSizeWithHeader(); } @@ -137,7 +133,7 @@ public class MultitenantVirtualBufferCache implements IVirtualBufferCache { @Override public int getNumPagesOfFile(int fileId) throws HyracksDataException { - return vbc.getNumPagesOfFile(fileId); + throw new UnsupportedOperationException(); } @Override @@ -209,28 +205,4 @@ public class MultitenantVirtualBufferCache implements IVirtualBufferCache { throws HyracksDataException { vbc.resizePage(page, multiplier, extraPageBlockHelper); } - - @Override - public String toString() { - return JSONUtil.fromMap(toMap()); - } - - private Map<String, Object> toMap() { - HashMap<String, Object> map = new HashMap<>(); - map.put("class", getClass().getSimpleName()); - map.put("vbc", vbc.toString()); - map.put("openCount", openCount); - return map; - } - - @Override - public int openFile(FileReference fileRef) throws HyracksDataException { - return vbc.openFile(fileRef); - } - - @Override - public void deleteFile(FileReference file) throws HyracksDataException { - vbc.deleteFile(file); - } - } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java index b6e2dd4..2a7bcca 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java @@ -25,6 +25,7 @@ import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; import org.apache.hyracks.storage.am.common.api.ITreeIndex; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public abstract class TreeIndexFactory<T extends ITreeIndex> extends IndexFactory<T> { @@ -33,10 +34,10 @@ public abstract class TreeIndexFactory<T extends ITreeIndex> extends IndexFactor protected final IBinaryComparatorFactory[] cmpFactories; protected final int fieldCount; - public TreeIndexFactory(IIOManager ioManager, IBufferCache bufferCache, IPageManagerFactory freePageManagerFactory, - ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory, - IBinaryComparatorFactory[] cmpFactories, int fieldCount) { - super(ioManager, bufferCache, freePageManagerFactory); + public TreeIndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider, + IPageManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory, + ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) { + super(ioManager, bufferCache, fileMapProvider, freePageManagerFactory); this.interiorFrameFactory = interiorFrameFactory; this.leafFrameFactory = leafFrameFactory; this.cmpFactories = cmpFactories; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java index d26677e..27d879c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java @@ -21,8 +21,6 @@ package org.apache.hyracks.storage.am.lsm.common.impls; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; import java.util.logging.Level; @@ -39,8 +37,7 @@ import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue; import org.apache.hyracks.storage.common.buffercache.VirtualPage; import org.apache.hyracks.storage.common.file.BufferedFileHandle; import org.apache.hyracks.storage.common.file.IFileMapManager; -import org.apache.hyracks.storage.common.file.FileMapManager; -import org.apache.hyracks.util.JSONUtil; +import org.apache.hyracks.storage.common.file.TransientFileMapManager; public class VirtualBufferCache implements IVirtualBufferCache { private static final Logger LOGGER = Logger.getLogger(ExternalIndexHarness.class.getName()); @@ -62,7 +59,7 @@ public class VirtualBufferCache implements IVirtualBufferCache { public VirtualBufferCache(ICacheMemoryAllocator allocator, int pageSize, int numPages) { this.allocator = allocator; - this.fileMapManager = new FileMapManager(); + this.fileMapManager = new TransientFileMapManager(); this.pageSize = pageSize; this.numPages = 2 * (numPages / 2) + 1; @@ -74,17 +71,12 @@ public class VirtualBufferCache implements IVirtualBufferCache { } @Override - public int createFile(FileReference fileRef) throws HyracksDataException { - return fileMapManager.registerFile(fileRef); - } - - @Override - public int openFile(FileReference fileRef) throws HyracksDataException { + public void createFile(FileReference fileRef) throws HyracksDataException { synchronized (fileMapManager) { if (fileMapManager.isMapped(fileRef)) { - return fileMapManager.lookupFileId(fileRef); + throw new HyracksDataException("File " + fileRef + " is already mapped"); } - return fileMapManager.registerFile(fileRef); + fileMapManager.registerFile(fileRef); } } @@ -97,16 +89,14 @@ public class VirtualBufferCache implements IVirtualBufferCache { } @Override - public void deleteFile(FileReference fileRef) throws HyracksDataException { + public void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException { synchronized (fileMapManager) { - int fileId = fileMapManager.lookupFileId(fileRef); - deleteFile(fileId); + if (!fileMapManager.isMapped(fileId)) { + throw new HyracksDataException("File with id " + fileId + " is not mapped"); + } + fileMapManager.unregisterFile(fileId); } - } - @Override - public void deleteFile(int fileId) throws HyracksDataException { - fileMapManager.unregisterFile(fileId); for (int i = 0; i < buckets.length; i++) { final CacheBucket bucket = buckets[i]; bucket.bucketLock.lock(); @@ -365,7 +355,12 @@ public class VirtualBufferCache implements IVirtualBufferCache { @Override public int getNumPagesOfFile(int fileId) throws HyracksDataException { - return -1; + synchronized (fileMapManager) { + if (fileMapManager.isMapped(fileId)) { + return numPages; + } + } + return 0; } @Override @@ -429,21 +424,7 @@ public class VirtualBufferCache implements IVirtualBufferCache { @Override public void purgeHandle(int fileId) throws HyracksDataException { - deleteFile(fileId); - } - @Override - public String toString() { - return JSONUtil.fromMap(toMap()); } - private Map<String, Object> toMap() { - HashMap<String, Object> map = new HashMap<>(); - map.put("class", getClass().getSimpleName()); - map.put("allocator", allocator.toString()); - map.put("pageSize", pageSize); - map.put("numPages", numPages); - map.put("open", open); - return map; - } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml index b94ebee..0f4063a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml @@ -16,8 +16,7 @@ ! specific language governing permissions and limitations ! under the License. !--> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <artifactId>hyracks-storage-am-lsm-invertedindex</artifactId> <parent> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java index eceb7b4..7258076 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java @@ -39,13 +39,4 @@ public interface IInvertedIndex extends IIndex { ITypeTraits[] getTokenTypeTraits(); IBinaryComparatorFactory[] getTokenCmpFactories(); - - /** - * Purge the index files out of the buffer cache. - * Can only be called if the caller is absolutely sure the files don't contain dirty pages - * - * @throws HyracksDataException - * if the index is active - */ - void purge() throws HyracksDataException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java index 7e2ec50..45ca106 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java @@ -42,6 +42,7 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokeniz import org.apache.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexUtils; import org.apache.hyracks.storage.common.IStorageManager; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public class LSMInvertedIndexLocalResource extends LsmResource { @@ -85,23 +86,25 @@ public class LSMInvertedIndexLocalResource extends LsmResource { FileReference file = ioManager.resolve(path); List<IVirtualBufferCache> virtualBufferCaches = vbcProvider.getVirtualBufferCaches(serviceCtx, file); IBufferCache bufferCache = storageManager.getBufferCache(serviceCtx); + IFileMapProvider fileMapManager = storageManager.getFileMapProvider(serviceCtx); ILSMMergePolicy mergePolicy = mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx); ILSMIOOperationScheduler ioScheduler = ioSchedulerProvider.getIoScheduler(serviceCtx); if (isPartitioned) { - return InvertedIndexUtils.createPartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, typeTraits, - cmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache, + return InvertedIndexUtils.createPartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, fileMapManager, + typeTraits, cmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache, file.getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy, opTrackerProvider.getOperationTracker(serviceCtx), ioScheduler, ioOpCallbackFactory.createIoOpCallback(), invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable, metadataPageManagerFactory); } else { - return InvertedIndexUtils.createLSMInvertedIndex(ioManager, virtualBufferCaches, typeTraits, cmpFactories, - tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache, file.getAbsolutePath(), - bloomFilterFalsePositiveRate, mergePolicy, opTrackerProvider.getOperationTracker(serviceCtx), - ioScheduler, ioOpCallbackFactory.createIoOpCallback(), invertedIndexFields, filterTypeTraits, - filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, - invertedIndexFieldsForNonBulkLoadOps, durable, metadataPageManagerFactory); + return InvertedIndexUtils.createLSMInvertedIndex(ioManager, virtualBufferCaches, fileMapManager, typeTraits, + cmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache, + file.getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy, + opTrackerProvider.getOperationTracker(serviceCtx), ioScheduler, + ioOpCallbackFactory.createIoOpCallback(), invertedIndexFields, filterTypeTraits, filterCmpFactories, + filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable, + metadataPageManagerFactory); } } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java index f3f2ff0..d5edbb5 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java @@ -87,6 +87,7 @@ import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.ISearchPredicate; import org.apache.hyracks.storage.common.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex { private static final Logger LOGGER = Logger.getLogger(LSMInvertedIndex.class.getName()); @@ -109,14 +110,14 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory, BloomFilterFactory bloomFilterFactory, IComponentFilterHelper filterHelper, ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, - double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager, ITypeTraits[] invListTypeTraits, - IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, - IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, - ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, - ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, int[] filterFields, - int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) - throws HyracksDataException { - super(ioManager, virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager, + double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, + ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, + ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, + IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, + ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, + int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, + boolean durable) throws HyracksDataException { + super(ioManager, virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory, filterManager, filterFields, durable, filterHelper, invertedIndexFields); this.tokenizerFactory = tokenizerFactory; @@ -133,10 +134,10 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) { InMemoryInvertedIndex memInvIndex = createInMemoryInvertedIndex(virtualBufferCache, new VirtualFreePageManager(virtualBufferCache), i); - BTree deleteKeysBTree = - BTreeUtils.createBTree(virtualBufferCache, new VirtualFreePageManager(virtualBufferCache), - invListTypeTraits, invListCmpFactories, BTreeLeafFrameType.REGULAR_NSM, - ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_del_" + i)); + BTree deleteKeysBTree = BTreeUtils.createBTree(virtualBufferCache, + new VirtualFreePageManager(virtualBufferCache), virtualBufferCache.getFileMapProvider(), + invListTypeTraits, invListCmpFactories, BTreeLeafFrameType.REGULAR_NSM, + ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_del_" + i)); LSMInvertedIndexMemoryComponent mutableComponent = new LSMInvertedIndexMemoryComponent(memInvIndex, deleteKeysBTree, virtualBufferCache, i == 0 ? true : false, filterHelper == null ? null : filterHelper.createFilter()); @@ -176,11 +177,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException { LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c; component.getBloomFilter().deactivate(); - component.getBloomFilter().purge(); component.getInvIndex().deactivate(); - component.getInvIndex().purge(); component.getDeletedKeysBTree().deactivate(); - component.getDeletedKeysBTree().purge(); } @Override @@ -191,6 +189,13 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex component.getBloomFilter().destroy(); } + @Override + protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { + LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c; + mutableComponent.getInvIndex().destroy(); + mutableComponent.getDeletedKeysBTree().destroy(); + } + /** * The keys in the in-memory deleted-keys BTree only refer to on-disk components. * We delete documents from the in-memory inverted index by deleting its entries directly, @@ -736,9 +741,4 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getDeleteIndexFileReference(), mergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir()); } - - @Override - public void purge() throws HyracksDataException { - throw new UnsupportedOperationException(); - } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java index 9254c45..aeb1e16 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java @@ -34,6 +34,7 @@ import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManage import org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNameMapper; +import org.apache.hyracks.storage.common.file.IFileMapProvider; // TODO: Refactor for better code sharing with other file managers. public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager implements IInvertedIndexFileNameMapper { @@ -65,32 +66,33 @@ public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager imp } }; - public LSMInvertedIndexFileManager(IIOManager ioManager, FileReference file, BTreeFactory btreeFactory) { - super(ioManager, file, null); + public LSMInvertedIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file, + BTreeFactory btreeFactory) { + super(ioManager, fileMapProvider, file, null); this.btreeFactory = btreeFactory; } @Override public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException { String ts = getCurrentTimestamp(); - String baseName = baseDir + ts + DELIMITER + ts; + String baseName = baseDir + ts + SPLIT_STRING + ts; // Begin timestamp and end timestamp are identical since it is a flush - return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + DICT_BTREE_SUFFIX), - createFlushFile(baseName + DELIMITER + DELETED_KEYS_BTREE_SUFFIX), - createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); + return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX), + createFlushFile(baseName + SPLIT_STRING + DELETED_KEYS_BTREE_SUFFIX), + createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING)); } @Override public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName) throws HyracksDataException { - String[] firstTimestampRange = firstFileName.split(DELIMITER); - String[] lastTimestampRange = lastFileName.split(DELIMITER); + String[] firstTimestampRange = firstFileName.split(SPLIT_STRING); + String[] lastTimestampRange = lastFileName.split(SPLIT_STRING); - String baseName = baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]; + String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1]; // Get the range of timestamps by taking the earliest and the latest timestamps - return new LSMComponentFileReferences(createMergeFile(baseName + DELIMITER + DICT_BTREE_SUFFIX), - createMergeFile(baseName + DELIMITER + DELETED_KEYS_BTREE_SUFFIX), - createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); + return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX), + createMergeFile(baseName + SPLIT_STRING + DELETED_KEYS_BTREE_SUFFIX), + createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING)); } @Override @@ -105,7 +107,7 @@ public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager imp cleanupAndGetValidFilesInternal(deletedKeysBTreeFilter, btreeFactory, allDeletedKeysBTreeFiles); HashSet<String> deletedKeysBTreeFilesSet = new HashSet<>(); for (ComparableFileName cmpFileName : allDeletedKeysBTreeFiles) { - int index = cmpFileName.fileName.lastIndexOf(DELIMITER); + int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING); deletedKeysBTreeFilesSet.add(cmpFileName.fileName.substring(0, index)); } @@ -208,8 +210,8 @@ public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager imp @Override public String getInvListsFilePath(String dictBTreeFilePath) { - int index = dictBTreeFilePath.lastIndexOf(DELIMITER); + int index = dictBTreeFilePath.lastIndexOf(SPLIT_STRING); String file = dictBTreeFilePath.substring(0, index); - return file + DELIMITER + INVLISTS_SUFFIX; + return file + SPLIT_STRING + INVLISTS_SUFFIX; } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java index d154356..8863427 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java @@ -41,6 +41,7 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInverted import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexUtils; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public class PartitionedLSMInvertedIndex extends LSMInvertedIndex { @@ -48,17 +49,17 @@ public class PartitionedLSMInvertedIndex extends LSMInvertedIndex { OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory, BloomFilterFactory bloomFilterFactory, IComponentFilterHelper filterHelper, ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, - double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager, ITypeTraits[] invListTypeTraits, - IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, - IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, - ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, - ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, int[] filterFields, - int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) - throws HyracksDataException { + double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, + ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, + ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, + IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, + ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, + int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, + boolean durable) throws HyracksDataException { super(ioManager, virtualBufferCaches, diskInvIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, fileManager, - invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, - mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, + diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, + tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java index d686634..4b673e5 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java @@ -31,6 +31,7 @@ import org.apache.hyracks.storage.am.btree.util.BTreeUtils; import org.apache.hyracks.storage.am.common.api.IIndexOperationContext; import org.apache.hyracks.storage.am.common.api.IPageManager; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; +import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor; import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory; @@ -73,8 +74,9 @@ public class InMemoryInvertedIndex implements IInvertedIndex { btreeTypeTraits[tokenTypeTraits.length + i] = invListTypeTraits[i]; btreeCmpFactories[tokenTypeTraits.length + i] = invListCmpFactories[i]; } - this.btree = BTreeUtils.createBTree(virtualBufferCache, virtualFreePageManager, btreeTypeTraits, - btreeCmpFactories, BTreeLeafFrameType.REGULAR_NSM, btreeFileRef); + this.btree = BTreeUtils.createBTree(virtualBufferCache, virtualFreePageManager, + ((IVirtualBufferCache) virtualBufferCache).getFileMapProvider(), btreeTypeTraits, btreeCmpFactories, + BTreeLeafFrameType.REGULAR_NSM, btreeFileRef); } @Override @@ -217,9 +219,4 @@ public class InMemoryInvertedIndex implements IInvertedIndex { public int getNumOfFilterFields() { return 0; } - - @Override - public void purge() throws HyracksDataException { - btree.purge(); - } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java index 25e4201..277eb64 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java @@ -26,7 +26,6 @@ import java.nio.ByteBuffer; import org.apache.hyracks.api.context.IHyracksCommonContext; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ITypeTraits; -import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; @@ -61,6 +60,7 @@ import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue; import org.apache.hyracks.storage.common.file.BufferedFileHandle; +import org.apache.hyracks.storage.common.file.IFileMapProvider; /** * An inverted index consists of two files: 1. a file storing (paginated) @@ -94,6 +94,7 @@ public class OnDiskInvertedIndex implements IInvertedIndex { protected BTree btree; protected int rootPageId = 0; protected IBufferCache bufferCache; + protected IFileMapProvider fileMapProvider; protected int fileId = -1; protected final ITypeTraits[] invListTypeTraits; protected final IBinaryComparatorFactory[] invListCmpFactories; @@ -108,18 +109,21 @@ public class OnDiskInvertedIndex implements IInvertedIndex { protected boolean isOpen = false; protected boolean wasOpen = false; - public OnDiskInvertedIndex(IBufferCache bufferCache, IInvertedListBuilder invListBuilder, - ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, - ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference btreeFile, - FileReference invListsFile, IPageManagerFactory pageManagerFactory) throws HyracksDataException { + public OnDiskInvertedIndex(IBufferCache bufferCache, IFileMapProvider fileMapProvider, + IInvertedListBuilder invListBuilder, ITypeTraits[] invListTypeTraits, + IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, + IBinaryComparatorFactory[] tokenCmpFactories, FileReference btreeFile, FileReference invListsFile, + IPageManagerFactory pageManagerFactory) throws HyracksDataException { this.bufferCache = bufferCache; + this.fileMapProvider = fileMapProvider; this.invListBuilder = invListBuilder; this.invListTypeTraits = invListTypeTraits; this.invListCmpFactories = invListCmpFactories; this.tokenTypeTraits = tokenTypeTraits; this.tokenCmpFactories = tokenCmpFactories; - this.btree = BTreeUtils.createBTree(bufferCache, getBTreeTypeTraits(tokenTypeTraits), tokenCmpFactories, - BTreeLeafFrameType.REGULAR_NSM, btreeFile, pageManagerFactory.createPageManager(bufferCache)); + this.btree = BTreeUtils.createBTree(bufferCache, fileMapProvider, getBTreeTypeTraits(tokenTypeTraits), + tokenCmpFactories, BTreeLeafFrameType.REGULAR_NSM, btreeFile, + pageManagerFactory.createPageManager(bufferCache)); this.numTokenFields = btree.getComparatorFactories().length; this.numInvListKeys = invListCmpFactories.length; this.invListsFile = invListsFile; @@ -135,7 +139,25 @@ public class OnDiskInvertedIndex implements IInvertedIndex { throw new HyracksDataException("Failed to create since index is already open."); } btree.create(); - fileId = bufferCache.createFile(invListsFile); + boolean fileIsMapped = false; + synchronized (fileMapProvider) { + fileIsMapped = fileMapProvider.isMapped(invListsFile); + if (!fileIsMapped) { + bufferCache.createFile(invListsFile); + } + fileId = fileMapProvider.lookupFileId(invListsFile); + try { + // Also creates the file if it doesn't exist yet. + bufferCache.openFile(fileId); + } catch (HyracksDataException e) { + // Revert state of buffer cache since file failed to open. + if (!fileIsMapped) { + bufferCache.deleteFile(fileId, false); + } + throw e; + } + } + bufferCache.closeFile(fileId); } @Override @@ -144,11 +166,25 @@ public class OnDiskInvertedIndex implements IInvertedIndex { throw new HyracksDataException("Failed to activate the index since it is already activated."); } btree.activate(); - if (fileId >= 0) { - bufferCache.openFile(fileId); - } else { - fileId = bufferCache.openFile(invListsFile); + boolean fileIsMapped = false; + synchronized (fileMapProvider) { + fileIsMapped = fileMapProvider.isMapped(invListsFile); + if (!fileIsMapped) { + bufferCache.createFile(invListsFile); + } + fileId = fileMapProvider.lookupFileId(invListsFile); + try { + // Also creates the file if it doesn't exist yet. + bufferCache.openFile(fileId); + } catch (HyracksDataException e) { + // Revert state of buffer cache since file failed to open. + if (!fileIsMapped) { + bufferCache.deleteFile(fileId, false); + } + throw e; + } } + isOpen = true; wasOpen = true; } @@ -158,8 +194,10 @@ public class OnDiskInvertedIndex implements IInvertedIndex { if (!isOpen && wasOpen) { throw new HyracksDataException("Failed to deactivate the index since it is already deactivated."); } + btree.deactivate(); bufferCache.closeFile(fileId); + isOpen = false; } @@ -168,8 +206,15 @@ public class OnDiskInvertedIndex implements IInvertedIndex { if (isOpen) { throw new HyracksDataException("Failed to destroy since index is already open."); } + btree.destroy(); - bufferCache.deleteFile(invListsFile); + invListsFile.delete(); + if (fileId == -1) { + return; + } + + bufferCache.deleteFile(fileId, false); + fileId = -1; } @Override @@ -179,9 +224,27 @@ public class OnDiskInvertedIndex implements IInvertedIndex { } btree.clear(); bufferCache.closeFile(fileId); - bufferCache.deleteFile(fileId); - fileId = bufferCache.createFile(invListsFile); - bufferCache.openFile(fileId); + bufferCache.deleteFile(fileId, false); + invListsFile.getFile().delete(); + + boolean fileIsMapped = false; + synchronized (fileMapProvider) { + fileIsMapped = fileMapProvider.isMapped(invListsFile); + if (!fileIsMapped) { + bufferCache.createFile(invListsFile); + } + fileId = fileMapProvider.lookupFileId(invListsFile); + try { + // Also creates the file if it doesn't exist yet. + bufferCache.openFile(fileId); + } catch (HyracksDataException e) { + // Revert state of buffer cache since file failed to open. + if (!fileIsMapped) { + bufferCache.deleteFile(fileId, false); + } + throw e; + } + } } @Override @@ -634,14 +697,4 @@ public class OnDiskInvertedIndex implements IInvertedIndex { public int getNumOfFilterFields() { return 0; } - - @Override - public synchronized void purge() throws HyracksDataException { - if (isOpen) { - throw HyracksDataException.create(ErrorCode.CANNOT_PURGE_ACTIVE_INDEX); - } - btree.purge(); - bufferCache.purgeHandle(fileId); - fileId = -1; - } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java index 70fcd03..dacef3c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java @@ -30,6 +30,7 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNam import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilderFactory; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public class OnDiskInvertedIndexFactory extends IndexFactory<IInvertedIndex> { @@ -40,12 +41,12 @@ public class OnDiskInvertedIndexFactory extends IndexFactory<IInvertedIndex> { protected final IBinaryComparatorFactory[] tokenCmpFactories; protected final IInvertedIndexFileNameMapper fileNameMapper; - public OnDiskInvertedIndexFactory(IIOManager ioManager, IBufferCache bufferCache, + public OnDiskInvertedIndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider, IInvertedListBuilderFactory invListBuilderFactory, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, IInvertedIndexFileNameMapper fileNameMapper, IPageManagerFactory pageManagerFactory) { - super(ioManager, bufferCache, pageManagerFactory); + super(ioManager, bufferCache, fileMapProvider, pageManagerFactory); this.invListBuilderFactory = invListBuilderFactory; this.invListTypeTraits = invListTypeTraits; this.invListCmpFactories = invListCmpFactories; @@ -59,7 +60,8 @@ public class OnDiskInvertedIndexFactory extends IndexFactory<IInvertedIndex> { String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getAbsolutePath()); FileReference invListsFile = ioManager.resolveAbsolutePath(invListsFilePath); IInvertedListBuilder invListBuilder = invListBuilderFactory.create(); - return new OnDiskInvertedIndex(bufferCache, invListBuilder, invListTypeTraits, invListCmpFactories, - tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile, freePageManagerFactory); + return new OnDiskInvertedIndex(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits, + invListCmpFactories, tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile, + freePageManagerFactory); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java index 0c0110e..452d59b 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java @@ -39,17 +39,19 @@ import org.apache.hyracks.storage.common.IIndexAccessor; import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public class PartitionedOnDiskInvertedIndex extends OnDiskInvertedIndex implements IPartitionedInvertedIndex { protected final int PARTITIONING_NUM_TOKENS_FIELD = 1; - public PartitionedOnDiskInvertedIndex(IBufferCache bufferCache, IInvertedListBuilder invListBuilder, - ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, - ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference btreeFile, - FileReference invListsFile, IPageManagerFactory pageManagerFactory) throws HyracksDataException { - super(bufferCache, invListBuilder, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, - btreeFile, invListsFile, pageManagerFactory); + public PartitionedOnDiskInvertedIndex(IBufferCache bufferCache, IFileMapProvider fileMapProvider, + IInvertedListBuilder invListBuilder, ITypeTraits[] invListTypeTraits, + IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, + IBinaryComparatorFactory[] tokenCmpFactories, FileReference btreeFile, FileReference invListsFile, + IPageManagerFactory pageManagerFactory) throws HyracksDataException { + super(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits, invListCmpFactories, tokenTypeTraits, + tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory); } public class PartitionedOnDiskInvertedIndexAccessor extends OnDiskInvertedIndexAccessor { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java index 5b2888a..ebc0152 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java @@ -29,16 +29,17 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNam import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilderFactory; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public class PartitionedOnDiskInvertedIndexFactory extends OnDiskInvertedIndexFactory { public PartitionedOnDiskInvertedIndexFactory(IIOManager ioManager, IBufferCache bufferCache, - IInvertedListBuilderFactory invListBuilderFactory, ITypeTraits[] invListTypeTraits, - IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, - IBinaryComparatorFactory[] tokenCmpFactories, IInvertedIndexFileNameMapper fileNameMapper, - IPageManagerFactory pageManagerFactory) { - super(ioManager, bufferCache, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits, - tokenCmpFactories, fileNameMapper, pageManagerFactory); + IFileMapProvider fileMapProvider, IInvertedListBuilderFactory invListBuilderFactory, + ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, + ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, + IInvertedIndexFileNameMapper fileNameMapper, IPageManagerFactory pageManagerFactory) { + super(ioManager, bufferCache, fileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, + tokenTypeTraits, tokenCmpFactories, fileNameMapper, pageManagerFactory); } @Override @@ -46,7 +47,8 @@ public class PartitionedOnDiskInvertedIndexFactory extends OnDiskInvertedIndexFa String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getAbsolutePath()); FileReference invListsFile = ioManager.resolveAbsolutePath(invListsFilePath); IInvertedListBuilder invListBuilder = invListBuilderFactory.create(); - return new PartitionedOnDiskInvertedIndex(bufferCache, invListBuilder, invListTypeTraits, invListCmpFactories, - tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile, freePageManagerFactory); + return new PartitionedOnDiskInvertedIndex(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits, + invListCmpFactories, tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile, + freePageManagerFactory); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java index 540e100..8a36f61 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java @@ -59,6 +59,7 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.PartitionedOnDiskI import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.PartitionedOnDiskInvertedIndexFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public class InvertedIndexUtils { @@ -81,23 +82,25 @@ public class InvertedIndexUtils { } public static OnDiskInvertedIndex createOnDiskInvertedIndex(IIOManager ioManager, IBufferCache bufferCache, - ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, - ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile, + IFileMapProvider fileMapProvider, ITypeTraits[] invListTypeTraits, + IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, + IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile, IPageManagerFactory pageManagerFactory) throws HyracksDataException { IInvertedListBuilder builder = new FixedSizeElementInvertedListBuilder(invListTypeTraits); FileReference btreeFile = getBTreeFile(ioManager, invListsFile); - return new OnDiskInvertedIndex(bufferCache, builder, invListTypeTraits, invListCmpFactories, tokenTypeTraits, - tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory); + return new OnDiskInvertedIndex(bufferCache, fileMapProvider, builder, invListTypeTraits, invListCmpFactories, + tokenTypeTraits, tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory); } public static PartitionedOnDiskInvertedIndex createPartitionedOnDiskInvertedIndex(IIOManager ioManager, - IBufferCache bufferCache, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, - ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile, + IBufferCache bufferCache, IFileMapProvider fileMapProvider, ITypeTraits[] invListTypeTraits, + IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, + IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile, IPageManagerFactory pageManagerFactory) throws HyracksDataException { IInvertedListBuilder builder = new FixedSizeElementInvertedListBuilder(invListTypeTraits); FileReference btreeFile = getBTreeFile(ioManager, invListsFile); - return new PartitionedOnDiskInvertedIndex(bufferCache, builder, invListTypeTraits, invListCmpFactories, - tokenTypeTraits, tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory); + return new PartitionedOnDiskInvertedIndex(bufferCache, fileMapProvider, builder, invListTypeTraits, + invListCmpFactories, tokenTypeTraits, tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory); } public static FileReference getBTreeFile(IIOManager ioManager, FileReference invListsFile) @@ -105,46 +108,49 @@ public class InvertedIndexUtils { return ioManager.resolveAbsolutePath(invListsFile.getFile().getPath() + "_btree"); } - public static BTreeFactory createDeletedKeysBTreeFactory(IIOManager ioManager, ITypeTraits[] invListTypeTraits, - IBinaryComparatorFactory[] invListCmpFactories, IBufferCache diskBufferCache, - IPageManagerFactory freePageManagerFactory) throws HyracksDataException { + public static BTreeFactory createDeletedKeysBTreeFactory(IIOManager ioManager, IFileMapProvider diskFileMapProvider, + ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, + IBufferCache diskBufferCache, IPageManagerFactory freePageManagerFactory) throws HyracksDataException { TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(invListTypeTraits); ITreeIndexFrameFactory leafFrameFactory = BTreeUtils.getLeafFrameFactory(tupleWriterFactory, BTreeLeafFrameType.REGULAR_NSM); ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory); - return new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, interiorFrameFactory, - leafFrameFactory, invListCmpFactories, invListCmpFactories.length); + BTreeFactory deletedKeysBTreeFactory = + new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, + interiorFrameFactory, leafFrameFactory, invListCmpFactories, invListCmpFactories.length); + return deletedKeysBTreeFactory; } public static LSMInvertedIndex createLSMInvertedIndex(IIOManager ioManager, - List<IVirtualBufferCache> virtualBufferCaches, ITypeTraits[] invListTypeTraits, - IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, - IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, - IBufferCache diskBufferCache, String absoluteOnDiskDir, double bloomFilterFalsePositiveRate, - ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, - ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, - IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, - int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, IMetadataPageManagerFactory pageManagerFactory) - throws HyracksDataException { + List<IVirtualBufferCache> virtualBufferCaches, IFileMapProvider diskFileMapProvider, + ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, + ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, + IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String absoluteOnDiskDir, + double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, + ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, + ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, + int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, + IMetadataPageManagerFactory pageManagerFactory) throws HyracksDataException { - BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, invListTypeTraits, - invListCmpFactories, diskBufferCache, pageManagerFactory); + BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider, + invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory); int[] bloomFilterKeyFields = new int[invListCmpFactories.length]; for (int i = 0; i < invListCmpFactories.length; i++) { bloomFilterKeyFields[i] = i; } - BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields); + BloomFilterFactory bloomFilterFactory = + new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields); FileReference onDiskDirFileRef = ioManager.resolveAbsolutePath(absoluteOnDiskDir); - LSMInvertedIndexFileManager fileManager = - new LSMInvertedIndexFileManager(ioManager, onDiskDirFileRef, deletedKeysBTreeFactory); + LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider, + onDiskDirFileRef, deletedKeysBTreeFactory); IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory(invListTypeTraits); - OnDiskInvertedIndexFactory invIndexFactory = - new OnDiskInvertedIndexFactory(ioManager, diskBufferCache, invListBuilderFactory, invListTypeTraits, - invListCmpFactories, tokenTypeTraits, tokenCmpFactories, fileManager, pageManagerFactory); + OnDiskInvertedIndexFactory invIndexFactory = new OnDiskInvertedIndexFactory(ioManager, diskBufferCache, + diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits, + tokenCmpFactories, fileManager, pageManagerFactory); ComponentFilterHelper filterHelper = null; LSMComponentFilterFrameFactory filterFrameFactory = null; @@ -157,40 +163,42 @@ public class InvertedIndexUtils { } return new LSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, - fileManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, - tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, - filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable); + fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, + tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, + invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, + durable); } public static PartitionedLSMInvertedIndex createPartitionedLSMInvertedIndex(IIOManager ioManager, - List<IVirtualBufferCache> virtualBufferCaches, ITypeTraits[] invListTypeTraits, - IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, - IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, - IBufferCache diskBufferCache, String absoluteOnDiskDir, double bloomFilterFalsePositiveRate, - ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, - ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, - IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, - int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, IPageManagerFactory pageManagerFactory) - throws HyracksDataException { + List<IVirtualBufferCache> virtualBufferCaches, IFileMapProvider diskFileMapProvider, + ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, + ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, + IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String absoluteOnDiskDir, + double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, + ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, + ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, + int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, + IPageManagerFactory pageManagerFactory) throws HyracksDataException { - BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, invListTypeTraits, - invListCmpFactories, diskBufferCache, pageManagerFactory); + BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider, + invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory); int[] bloomFilterKeyFields = new int[invListCmpFactories.length]; for (int i = 0; i < invListCmpFactories.length; i++) { bloomFilterKeyFields[i] = i; } - BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields); + BloomFilterFactory bloomFilterFactory = + new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields); FileReference onDiskDirFileRef = ioManager.resolveAbsolutePath(absoluteOnDiskDir); - LSMInvertedIndexFileManager fileManager = - new LSMInvertedIndexFileManager(ioManager, onDiskDirFileRef, deletedKeysBTreeFactory); + LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider, + onDiskDirFileRef, deletedKeysBTreeFactory); IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory(invListTypeTraits); PartitionedOnDiskInvertedIndexFactory invIndexFactory = new PartitionedOnDiskInvertedIndexFactory(ioManager, - diskBufferCache, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits, - tokenCmpFactories, fileManager, pageManagerFactory); + diskBufferCache, diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, + tokenTypeTraits, tokenCmpFactories, fileManager, pageManagerFactory); ComponentFilterHelper filterHelper = null; LSMComponentFilterFrameFactory filterFrameFactory = null; @@ -203,8 +211,9 @@ public class InvertedIndexUtils { } return new PartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, - fileManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, - tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, - filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable); + fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, + tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, + invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, + durable); } }
