http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java index f1ae6b0..9f648c5 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java @@ -19,10 +19,13 @@ package org.apache.hyracks.storage.am.common.build; import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; import org.apache.hyracks.api.application.INCServiceContext; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; +import org.apache.hyracks.api.util.IoUtil; import org.apache.hyracks.storage.am.common.api.IIndexBuilder; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; import org.apache.hyracks.storage.common.IIndex; @@ -35,6 +38,7 @@ import org.apache.hyracks.storage.common.LocalResource; import org.apache.hyracks.storage.common.file.IResourceIdFactory; public class IndexBuilder implements IIndexBuilder { + private static final Logger LOGGER = Logger.getLogger(IndexBuilder.class.getName()); protected final INCServiceContext ctx; protected final IStorageManager storageManager; @@ -81,8 +85,22 @@ public class IndexBuilder implements IIndexBuilder { //The reason for this is to handle many cases such as: //1. Crash while delete index is running (we don't do global cleanup on restart) //2. Node leaves and then join with old data + LOGGER.log(Level.WARNING, + "Removing existing index on index create for the index: " + resourceRef.getRelativePath()); lcManager.unregister(resourceRef.getRelativePath()); + index.destroy(); } else { + if (resourceRef.getFile().exists()) { + // Index is not registered but the index file exists + // This is another big problem that we need to disallow soon + // We can only disallow this if we have a global cleanup after crash + // on reboot + LOGGER.log(Level.WARNING, + "Deleting " + resourceRef.getRelativePath() + + " on index create. The index is not registered" + + " but the file exists in the filesystem"); + IoUtil.delete(resourceRef); + } index = resource.createInstance(ctx); } index.create();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java index c00cecb..cc3bfe7 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java @@ -37,7 +37,6 @@ import org.apache.hyracks.storage.am.common.util.TreeIndexStatsGatherer; import org.apache.hyracks.storage.common.IStorageManager; import org.apache.hyracks.storage.common.LocalResource; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public class TreeIndexStatsOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable { private final IHyracksTaskContext ctx; @@ -69,14 +68,11 @@ public class TreeIndexStatsOperatorNodePushable extends AbstractUnaryOutputSourc try { writer.open(); IBufferCache bufferCache = storageManager.getBufferCache(ctx.getJobletContext().getServiceContext()); - IFileMapProvider fileMapProvider = - storageManager.getFileMapProvider(ctx.getJobletContext().getServiceContext()); LocalResource resource = treeIndexHelper.getResource(); IIOManager ioManager = ctx.getIoManager(); FileReference fileRef = ioManager.resolve(resource.getPath()); - int indexFileId = fileMapProvider.lookupFileId(fileRef); TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, treeIndex.getPageManager(), - indexFileId, treeIndex.getRootPageId()); + fileRef, treeIndex.getRootPageId()); TreeIndexStats stats = statsGatherer.gatherStats(treeIndex.getLeafFrameFactory().createFrame(), treeIndex.getInteriorFrameFactory().createFrame(), treeIndex.getPageManager().createMetadataFrame()); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java index 163a3d1..8ffd0eb 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java @@ -40,7 +40,6 @@ import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue; import org.apache.hyracks.storage.common.file.BufferedFileHandle; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public abstract class AbstractTreeIndex implements ITreeIndex { @@ -49,7 +48,6 @@ public abstract class AbstractTreeIndex implements ITreeIndex { protected int rootPage = 1; protected final IBufferCache bufferCache; - protected final IFileMapProvider fileMapProvider; protected final IPageManager freePageManager; protected final ITreeIndexFrameFactory interiorFrameFactory; @@ -59,21 +57,16 @@ public abstract class AbstractTreeIndex implements ITreeIndex { protected final int fieldCount; protected FileReference file; - protected int fileId = -1; + private int fileId = -1; protected boolean isActive = false; - //hasEverBeenActivated is to stop the throwing of an exception of deactivating an index that - //was never activated or failed to activate in try/finally blocks, as there's no way to know if - //an index is activated or not from the outside. - protected boolean hasEverBeenActivated = false; protected int bulkloadLeafStart = 0; - public AbstractTreeIndex(IBufferCache bufferCache, IFileMapProvider fileMapProvider, IPageManager freePageManager, + public AbstractTreeIndex(IBufferCache bufferCache, IPageManager freePageManager, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount, FileReference file) { this.bufferCache = bufferCache; - this.fileMapProvider = fileMapProvider; this.freePageManager = freePageManager; this.interiorFrameFactory = interiorFrameFactory; this.leafFrameFactory = leafFrameFactory; @@ -87,9 +80,8 @@ public abstract class AbstractTreeIndex implements ITreeIndex { if (isActive) { throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_ACTIVE_INDEX); } - synchronized (fileMapProvider) { - fileId = createAndOpen(bufferCache, fileMapProvider, file); - } + fileId = bufferCache.createFile(file); + bufferCache.openFile(fileId); freePageManager.open(fileId); freePageManager.init(interiorFrameFactory, leafFrameFactory); setRootPage(); @@ -97,27 +89,6 @@ public abstract class AbstractTreeIndex implements ITreeIndex { bufferCache.closeFile(fileId); } - public static int createAndOpen(IBufferCache bufferCache, IFileMapProvider fileMapProvider, FileReference file) - throws HyracksDataException { - int fileId; - boolean fileIsMapped = fileMapProvider.isMapped(file); - if (!fileIsMapped) { - bufferCache.createFile(file); - } - fileId = fileMapProvider.lookupFileId(file); - try { - // Also creates the file if it doesn't exist yet. - bufferCache.openFile(fileId); - } catch (HyracksDataException e) { - // Revert state of buffer cache since file failed to open. - if (!fileIsMapped) { - bufferCache.deleteFile(fileId, false); - } - throw e; - } - return fileId; - } - private void setRootPage() throws HyracksDataException { rootPage = freePageManager.getRootPageId(); bulkloadLeafStart = freePageManager.getBulkLoadLeaf(); @@ -128,49 +99,35 @@ public abstract class AbstractTreeIndex implements ITreeIndex { if (isActive) { throw HyracksDataException.create(ErrorCode.CANNOT_ACTIVATE_ACTIVE_INDEX); } - boolean fileIsMapped = false; - synchronized (fileMapProvider) { - fileIsMapped = fileMapProvider.isMapped(file); - if (!fileIsMapped) { - bufferCache.createFile(file); - } - fileId = fileMapProvider.lookupFileId(file); - try { - // Also creates the file if it doesn't exist yet. - bufferCache.openFile(fileId); - } catch (HyracksDataException e) { - // Revert state of buffer cache since file failed to open. - if (!fileIsMapped) { - bufferCache.deleteFile(fileId, false); - } - throw e; - } + if (fileId >= 0) { + bufferCache.openFile(fileId); + } else { + fileId = bufferCache.openFile(file); } freePageManager.open(fileId); setRootPage(); // TODO: Should probably have some way to check that the tree is physically consistent // or that the file we just opened actually is a tree isActive = true; - hasEverBeenActivated = true; } @Override public synchronized void deactivate() throws HyracksDataException { - if (!isActive && hasEverBeenActivated) { + if (!isActive) { throw HyracksDataException.create(ErrorCode.CANNOT_DEACTIVATE_INACTIVE_INDEX); } - if (isActive) { - freePageManager.close(); - bufferCache.closeFile(fileId); - } - + freePageManager.close(); + bufferCache.closeFile(fileId); isActive = false; } - public synchronized void deactivateCloseHandle() throws HyracksDataException { - deactivate(); + public void purge() throws HyracksDataException { + if (isActive) { + throw HyracksDataException.create(ErrorCode.CANNOT_PURGE_ACTIVE_INDEX); + } bufferCache.purgeHandle(fileId); - + // after purging, the fileId has no mapping and no meaning + fileId = -1; } @Override @@ -178,13 +135,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex { if (isActive) { throw HyracksDataException.create(ErrorCode.CANNOT_DESTROY_ACTIVE_INDEX); } - - if (fileId == -1) { - return; - } - bufferCache.deleteFile(fileId, false); - file.delete(); - fileId = -1; + bufferCache.deleteFile(file); } @Override @@ -284,7 +235,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex { queue = bufferCache.createFIFOQueue(); if (!isEmptyTree(leafFrame)) { - throw new HyracksDataException("Cannot bulk-load a non-empty tree."); + throw HyracksDataException.create(ErrorCode.CANNOT_BULK_LOAD_NON_EMPTY_TREE); } this.cmp = MultiComparator.create(cmpFactories); @@ -312,9 +263,6 @@ public abstract class AbstractTreeIndex implements ITreeIndex { pagesToWrite = new ArrayList<>(); } - @Override - public abstract void add(ITupleReference tuple) throws HyracksDataException; - protected void handleException() throws HyracksDataException { // Unlatch and unpin pages that weren't in the queue to avoid leaking memory. for (NodeFrontier nodeFrontier : nodeFrontiers) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java index 2b47238..959f27a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java @@ -22,7 +22,7 @@ import java.util.ArrayList; import java.util.Random; import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.util.MathUtil; +import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.storage.am.common.api.IMetadataPageManager; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame; @@ -30,25 +30,25 @@ import org.apache.hyracks.storage.common.arraylist.IntArrayList; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.file.BufferedFileHandle; +import org.apache.hyracks.util.MathUtil; public class TreeIndexBufferCacheWarmup { private final IBufferCache bufferCache; private final IMetadataPageManager freePageManager; - private final int fileId; + private final FileReference fileRef; private final ArrayList<IntArrayList> pagesByLevel = new ArrayList<>(); private final Random rnd = new Random(); - public TreeIndexBufferCacheWarmup(IBufferCache bufferCache, - IMetadataPageManager freePageManager, int fileId) { + public TreeIndexBufferCacheWarmup(IBufferCache bufferCache, IMetadataPageManager freePageManager, + FileReference fileRef) { this.bufferCache = bufferCache; this.freePageManager = freePageManager; - this.fileId = fileId; + this.fileRef = fileRef; } - public void warmup(ITreeIndexFrame frame, - ITreeIndexMetadataFrame metaFrame, int[] warmupTreeLevels, + public void warmup(ITreeIndexFrame frame, ITreeIndexMetadataFrame metaFrame, int[] warmupTreeLevels, int[] warmupRepeats) throws HyracksDataException { - bufferCache.openFile(fileId); + int fileId = bufferCache.openFile(fileRef); // scan entire file to determine pages in each level int maxPageId = freePageManager.getMaxPageId(metaFrame); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java index 7e14b4a..5410282 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java @@ -19,6 +19,7 @@ package org.apache.hyracks.storage.am.common.util; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.storage.am.common.api.IPageManager; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame; @@ -31,25 +32,21 @@ public class TreeIndexStatsGatherer { private final TreeIndexStats treeIndexStats = new TreeIndexStats(); private final IBufferCache bufferCache; private final IPageManager freePageManager; - private final int fileId; + private final FileReference fileRef; private final int rootPage; - public TreeIndexStatsGatherer(IBufferCache bufferCache, - IPageManager freePageManager, int fileId, int rootPage) { + public TreeIndexStatsGatherer(IBufferCache bufferCache, IPageManager freePageManager, FileReference fileRef, + int rootPage) { this.bufferCache = bufferCache; this.freePageManager = freePageManager; - this.fileId = fileId; + this.fileRef = fileRef; this.rootPage = rootPage; } - public TreeIndexStats gatherStats(ITreeIndexFrame leafFrame, - ITreeIndexFrame interiorFrame, ITreeIndexMetadataFrame metaFrame) - throws HyracksDataException { - - bufferCache.openFile(fileId); - + public TreeIndexStats gatherStats(ITreeIndexFrame leafFrame, ITreeIndexFrame interiorFrame, + ITreeIndexMetadataFrame metaFrame) throws HyracksDataException { + int fileId = bufferCache.openFile(fileRef); treeIndexStats.begin(); - int maxPageId = freePageManager.getMaxPageId(metaFrame); for (int pageId = 0; pageId <= maxPageId; pageId++) { ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java index 6860b97..f1e9cf1 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java @@ -57,9 +57,9 @@ public class ExternalBTreeLocalResource extends LSMBTreeLocalResource { public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException { IIOManager ioManager = serviceCtx.getIoManager(); FileReference file = ioManager.resolve(path); - return LSMBTreeUtil.createExternalBTree(ioManager, file, storageManager.getBufferCache(serviceCtx), - storageManager.getFileMapProvider(serviceCtx), typeTraits, cmpFactories, bloomFilterKeyFields, - bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), + return LSMBTreeUtil.createExternalBTree(ioManager, file, storageManager.getBufferCache(serviceCtx), typeTraits, + cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, + mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), opTrackerProvider.getOperationTracker(serviceCtx), ioSchedulerProvider.getIoScheduler(serviceCtx), ioOpCallbackFactory.createIoOpCallback(), durable, metadataPageManagerFactory); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java index 3819066..317a479 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java @@ -61,7 +61,7 @@ public class ExternalBTreeWithBuddyLocalResource extends LSMBTreeLocalResource { IIOManager ioManager = serviceCtx.getIoManager(); FileReference file = ioManager.resolve(path); return LSMBTreeUtil.createExternalBTreeWithBuddy(ioManager, file, storageManager.getBufferCache(serviceCtx), - storageManager.getFileMapProvider(serviceCtx), typeTraits, cmpFactories, bloomFilterFalsePositiveRate, + typeTraits, cmpFactories, bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), opTrackerProvider.getOperationTracker(serviceCtx), ioSchedulerProvider.getIoScheduler(serviceCtx), ioOpCallbackFactory.createIoOpCallback(), bloomFilterKeyFields, durable, metadataPageManagerFactory); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java index 6416f6b..f6e311c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java @@ -70,9 +70,9 @@ public class LSMBTreeLocalResource extends LsmResource { IIOManager ioManager = serviceCtx.getIoManager(); FileReference file = ioManager.resolve(path); List<IVirtualBufferCache> vbcs = vbcProvider.getVirtualBufferCaches(serviceCtx, file); - return LSMBTreeUtil.createLSMTree(ioManager, vbcs, file, storageManager.getBufferCache(serviceCtx), - storageManager.getFileMapProvider(serviceCtx), typeTraits, cmpFactories, bloomFilterKeyFields, - bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), + return LSMBTreeUtil.createLSMTree(ioManager, vbcs, file, storageManager.getBufferCache(serviceCtx), typeTraits, + cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, + mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), opTrackerProvider.getOperationTracker(serviceCtx), ioSchedulerProvider.getIoScheduler(serviceCtx), ioOpCallbackFactory.createIoOpCallback(), isPrimary, filterTypeTraits, filterCmpFactories, btreeFields, filterFields, durable, metadataPageManagerFactory); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java index cdde692..c06edfd 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java @@ -62,7 +62,6 @@ import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.ISearchPredicate; -import org.apache.hyracks.storage.common.file.IFileMapProvider; /** * This is an lsm b-tree that does not have memory component and is modified @@ -93,13 +92,12 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager, TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory, BloomFilterFactory bloomFilterFactory, - double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount, - IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, - ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, + double bloomFilterFalsePositiveRate, IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, + ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, TreeIndexFactory<BTree> transactionBTreeFactory, boolean durable) { super(ioManager, insertLeafFrameFactory, deleteLeafFrameFactory, fileManager, diskBTreeFactory, - bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, fieldCount, - cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, false, durable); + bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, cmpFactories, mergePolicy, + opTracker, ioScheduler, ioOpCallback, false, durable); this.transactionComponentFactory = new LSMBTreeDiskComponentFactory(transactionBTreeFactory, bloomFilterFactory, null); this.secondDiskComponents = new LinkedList<>(); @@ -301,21 +299,13 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback); cb.afterFinalize(LSMOperationType.FLUSH, null); } - for (ILSMComponent c : diskComponents) { - LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; - BTree btree = component.getBTree(); - BloomFilter bloomFilter = component.getBloomFilter(); - btree.deactivateCloseHandle(); - bloomFilter.deactivate(); + for (ILSMDiskComponent c : diskComponents) { + deactivateDiskComponent(c); } - for (ILSMComponent c : secondDiskComponents) { - // Only deactivate non shared components (So components are not de-activated twice) + for (ILSMDiskComponent c : secondDiskComponents) { + // Only deactivate non shared components if (!diskComponents.contains(c)) { - LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; - BTree btree = component.getBTree(); - BloomFilter bloomFilter = component.getBloomFilter(); - btree.deactivateCloseHandle(); - bloomFilter.deactivate(); + deactivateDiskComponent(c); } } isActive = false; @@ -357,17 +347,13 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { if (isActive) { throw new HyracksDataException("Failed to destroy the index since it is activated."); } - for (ILSMComponent c : diskComponents) { - LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; - component.getBTree().destroy(); - component.getBloomFilter().destroy(); + for (ILSMDiskComponent c : diskComponents) { + destroyDiskComponent(c); // Remove from second list to avoid destroying twice secondDiskComponents.remove(c); } - for (ILSMComponent c : secondDiskComponents) { - LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; - component.getBTree().destroy(); - component.getBloomFilter().destroy(); + for (ILSMDiskComponent c : secondDiskComponents) { + destroyDiskComponent(c); } diskComponents.clear(); secondDiskComponents.clear(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java index 1c99b17..9959583 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java @@ -72,7 +72,6 @@ import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.ISearchPredicate; import org.apache.hyracks.storage.common.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeIndex, ITwoPCIndex { @@ -100,12 +99,12 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, TreeIndexFactory<BTree> bulkLoadBTreeFactory, TreeIndexFactory<BTree> copyBtreeFactory, TreeIndexFactory<BTree> buddyBtreeFactory, BloomFilterFactory bloomFilterFactory, - IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, - ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, + double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, + ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, IBinaryComparatorFactory[] btreeCmpFactories, IBinaryComparatorFactory[] buddyBtreeCmpFactories, int[] buddyBTreeFields, boolean durable) { - super(ioManager, diskBufferCache, fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy, - opTracker, ioScheduler, ioOpCallback, durable); + super(ioManager, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy, opTracker, + ioScheduler, ioOpCallback, durable); this.btreeCmpFactories = btreeCmpFactories; this.buddyBtreeCmpFactories = buddyBtreeCmpFactories; this.buddyBTreeFields = buddyBTreeFields; @@ -121,12 +120,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd @Override public void create() throws HyracksDataException { - if (isActive) { - throw new HyracksDataException("Failed to create the index since it is activated."); - } - fileManager.deleteDirs(); - fileManager.createDirs(); - diskComponents.clear(); + super.create(); secondDiskComponents.clear(); } @@ -773,14 +767,12 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd BTree btree = component.getBTree(); BTree buddyBtree = component.getBuddyBTree(); BloomFilter bloomFilter = component.getBloomFilter(); - btree.deactivateCloseHandle(); - buddyBtree.deactivateCloseHandle(); + btree.deactivate(); + buddyBtree.deactivate(); bloomFilter.deactivate(); - } - - @Override - protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { - //do nothing since external index never use memory components + btree.purge(); + buddyBtree.purge(); + bloomFilter.purge(); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java index e830b3e..a5c59d6 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java @@ -76,7 +76,6 @@ import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.ISearchPredicate; import org.apache.hyracks.storage.common.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { @@ -103,11 +102,11 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory, BloomFilterFactory bloomFilterFactory, IComponentFilterHelper filterHelper, ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, - double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount, - IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, - ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, - int[] btreeFields, int[] filterFields, boolean durable) throws HyracksDataException { - super(ioManager, virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider, + double bloomFilterFalsePositiveRate, int fieldCount, IBinaryComparatorFactory[] cmpFactories, + ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, + ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, int[] btreeFields, int[] filterFields, + boolean durable) throws HyracksDataException { + super(ioManager, virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory, filterManager, filterFields, durable, filterHelper, btreeFields); this.insertLeafFrameFactory = insertLeafFrameFactory; @@ -116,8 +115,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { int i = 0; for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) { LSMBTreeMemoryComponent mutableComponent = new LSMBTreeMemoryComponent( - new BTree(virtualBufferCache, virtualBufferCache.getFileMapProvider(), - new VirtualFreePageManager(virtualBufferCache), interiorFrameFactory, + new BTree(virtualBufferCache, new VirtualFreePageManager(virtualBufferCache), interiorFrameFactory, insertLeafFrameFactory, cmpFactories, fieldCount, ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_" + i)), virtualBufferCache, i == 0 ? true : false, @@ -137,19 +135,18 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager, TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory, BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate, - IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories, - ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, - ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, boolean durable) { - super(ioManager, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider, - bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, durable); + IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, + ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, + boolean durable) { + super(ioManager, diskBTreeFactory.getBufferCache(), fileManager, bloomFilterFalsePositiveRate, mergePolicy, + opTracker, ioScheduler, ioOpCallback, durable); this.insertLeafFrameFactory = insertLeafFrameFactory; this.deleteLeafFrameFactory = deleteLeafFrameFactory; this.cmpFactories = cmpFactories; - componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, null); - bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory, null); this.needKeyDupCheck = needKeyDupCheck; - //TODO remove BloomFilter from external dataset's secondary LSMBTree index this.hasBloomFilter = true; + componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, null); + bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory, null); } public boolean hasBloomFilter() { @@ -183,12 +180,6 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { } @Override - protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { - LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c; - mutableComponent.getBTree().destroy(); - } - - @Override protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException { LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; if (hasBloomFilter) { @@ -587,9 +578,11 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { @Override protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException { LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; - component.getBTree().deactivateCloseHandle(); + component.getBTree().deactivate(); + component.getBTree().purge(); if (hasBloomFilter) { component.getBloomFilter().deactivate(); + component.getBloomFilter().purge(); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java index 194eb3a..432a2b8 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java @@ -37,17 +37,15 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndex; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public class LSMBTreeFileManager extends AbstractLSMIndexFileManager { - public static final String BTREE_STRING = "b"; private final TreeIndexFactory<? extends ITreeIndex> btreeFactory; private final boolean hasBloomFilter; - public LSMBTreeFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file, + public LSMBTreeFileManager(IIOManager ioManager, FileReference file, TreeIndexFactory<? extends ITreeIndex> btreeFactory, boolean hasBloomFilter) { - super(ioManager, fileMapProvider, file, null); + super(ioManager, file, null); this.btreeFactory = btreeFactory; this.hasBloomFilter = hasBloomFilter; } @@ -55,28 +53,28 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager { @Override public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException { String ts = getCurrentTimestamp(); - String baseName = baseDir + ts + SPLIT_STRING + ts; + String baseName = baseDir + ts + DELIMITER + ts; // Begin timestamp and end timestamp are identical since it is a flush - return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), null, - hasBloomFilter ? createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING) : null); + return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), null, + hasBloomFilter ? createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX) : null); } @Override public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName) throws HyracksDataException { - String[] firstTimestampRange = firstFileName.split(SPLIT_STRING); - String[] lastTimestampRange = lastFileName.split(SPLIT_STRING); + String[] firstTimestampRange = firstFileName.split(DELIMITER); + String[] lastTimestampRange = lastFileName.split(DELIMITER); - String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1]; + String baseName = baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]; // Get the range of timestamps by taking the earliest and the latest timestamps - return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + BTREE_STRING), null, - hasBloomFilter ? createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING) : null); + return new LSMComponentFileReferences(createMergeFile(baseName + DELIMITER + BTREE_SUFFIX), null, + hasBloomFilter ? createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX) : null); } private static FilenameFilter btreeFilter = new FilenameFilter() { @Override public boolean accept(File dir, String name) { - return !name.startsWith(".") && name.endsWith(BTREE_STRING); + return !name.startsWith(".") && name.endsWith(BTREE_SUFFIX); } }; @@ -95,7 +93,7 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager { cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, btreeFilter), btreeFactory, allBTreeFiles); HashSet<String> btreeFilesSet = new HashSet<>(); for (ComparableFileName cmpFileName : allBTreeFiles) { - int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING); + int index = cmpFileName.fileName.lastIndexOf(DELIMITER); btreeFilesSet.add(cmpFileName.fileName.substring(0, index)); } @@ -200,19 +198,19 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager { public LSMComponentFileReferences getNewTransactionFileReference() throws IOException { String ts = getCurrentTimestamp(); // Create transaction lock file - Files.createFile(Paths.get(baseDir + TRANSACTION_PREFIX + ts)); + Files.createFile(Paths.get(baseDir + TXN_PREFIX + ts)); - String baseName = baseDir + ts + SPLIT_STRING + ts; + String baseName = baseDir + ts + DELIMITER + ts; // Begin timestamp and end timestamp are identical since it is a transaction - return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), null, - createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING)); + return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), null, + createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); } @Override public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException { FilenameFilter transactionFilter; File dir = new File(baseDir); - String[] files = dir.list(transactionFileNameFilter); + String[] files = dir.list(txnFileNameFilter); if (files.length == 0) { return null; } @@ -236,9 +234,9 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager { File bloomFilterFile = null; for (String fileName : files) { - if (fileName.endsWith(BTREE_STRING)) { + if (fileName.endsWith(BTREE_SUFFIX)) { bTreeFile = new File(dir.getPath() + File.separator + fileName); - } else if (fileName.endsWith(BLOOM_FILTER_STRING)) { + } else if (fileName.endsWith(BLOOM_FILTER_SUFFIX)) { bloomFilterFile = new File(dir.getPath() + File.separator + fileName); } else { throw new HyracksDataException("unrecognized file found = " + fileName); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java index 0173c06..c866a45 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java @@ -36,11 +36,8 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndex; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { - public static final String BUDDY_BTREE_STRING = "buddy"; - public static final String BTREE_STRING = "b"; private final TreeIndexFactory<? extends ITreeIndex> btreeFactory; private final TreeIndexFactory<? extends ITreeIndex> buddyBtreeFactory; @@ -48,21 +45,21 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { private static FilenameFilter btreeFilter = new FilenameFilter() { @Override public boolean accept(File dir, String name) { - return !name.startsWith(".") && name.endsWith(BTREE_STRING); + return !name.startsWith(".") && name.endsWith(BTREE_SUFFIX); } }; private static FilenameFilter buddyBtreeFilter = new FilenameFilter() { @Override public boolean accept(File dir, String name) { - return !name.startsWith(".") && name.endsWith(BUDDY_BTREE_STRING); + return !name.startsWith(".") && name.endsWith(DELETE_TREE_SUFFIX); } }; - public LSMBTreeWithBuddyFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file, + public LSMBTreeWithBuddyFileManager(IIOManager ioManager, FileReference file, TreeIndexFactory<? extends ITreeIndex> btreeFactory, TreeIndexFactory<? extends ITreeIndex> buddyBtreeFactory) { - super(ioManager, fileMapProvider, file, null); + super(ioManager, file, null); this.buddyBtreeFactory = buddyBtreeFactory; this.btreeFactory = btreeFactory; } @@ -70,25 +67,25 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { @Override public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException { String ts = getCurrentTimestamp(); - String baseName = baseDir + ts + SPLIT_STRING + ts; + String baseName = baseDir + ts + DELIMITER + ts; // Begin timestamp and end timestamp are identical since it is a flush - return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), - createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING), - createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING)); + return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), + createFlushFile(baseName + DELIMITER + DELETE_TREE_SUFFIX), + createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); } @Override public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName) throws HyracksDataException { - String[] firstTimestampRange = firstFileName.split(SPLIT_STRING); - String[] lastTimestampRange = lastFileName.split(SPLIT_STRING); + String[] firstTimestampRange = firstFileName.split(DELIMITER); + String[] lastTimestampRange = lastFileName.split(DELIMITER); - String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1]; + String baseName = baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]; // Get the range of timestamps by taking the earliest and the latest // timestamps - return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + BTREE_STRING), - createMergeFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING), - createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING)); + return new LSMComponentFileReferences(createMergeFile(baseName + DELIMITER + BTREE_SUFFIX), + createMergeFile(baseName + DELIMITER + DELETE_TREE_SUFFIX), + createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); } @Override @@ -106,7 +103,7 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { allBTreeFiles); HashSet<String> btreeFilesSet = new HashSet<>(); for (ComparableFileName cmpFileName : allBTreeFiles) { - int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING); + int index = cmpFileName.fileName.lastIndexOf(DELIMITER); btreeFilesSet.add(cmpFileName.fileName.substring(0, index)); } validateFiles(btreeFilesSet, allBuddyBTreeFiles, getCompoundFilter(buddyBtreeFilter, transactionFilefilter), @@ -206,19 +203,19 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { public LSMComponentFileReferences getNewTransactionFileReference() throws IOException { String ts = getCurrentTimestamp(); // Create transaction lock file - Files.createFile(Paths.get(baseDir + TRANSACTION_PREFIX + ts)); + Files.createFile(Paths.get(baseDir + TXN_PREFIX + ts)); - String baseName = baseDir + ts + SPLIT_STRING + ts; - return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), - createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING), - createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING)); + String baseName = baseDir + ts + DELIMITER + ts; + return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), + createFlushFile(baseName + DELIMITER + DELETE_TREE_SUFFIX), + createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); } @Override public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException { FilenameFilter transactionFilter; File dir = new File(baseDir); - String[] files = dir.list(transactionFileNameFilter); + String[] files = dir.list(txnFileNameFilter); if (files.length == 0) { return null; } @@ -243,11 +240,11 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { File buddyBTreeFile = null; File bloomFilterFile = null; for (String fileName : files) { - if (fileName.endsWith(BTREE_STRING)) { + if (fileName.endsWith(BTREE_SUFFIX)) { bTreeFile = new File(dir.getPath() + File.separator + fileName); - } else if (fileName.endsWith(BUDDY_BTREE_STRING)) { + } else if (fileName.endsWith(DELETE_TREE_SUFFIX)) { buddyBTreeFile = new File(dir.getPath() + File.separator + fileName); - } else if (fileName.endsWith(BLOOM_FILTER_STRING)) { + } else if (fileName.endsWith(BLOOM_FILTER_SUFFIX)) { bloomFilterFile = new File(dir.getPath() + File.separator + fileName); } else { throw new HyracksDataException("unrecognized file found = " + fileName); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java index 259f25f..792b872 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java @@ -52,7 +52,6 @@ import org.apache.hyracks.storage.am.lsm.common.impls.ComponentFilterHelper; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager; import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public class LSMBTreeUtil { @@ -60,13 +59,12 @@ public class LSMBTreeUtil { } public static LSMBTree createLSMTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, - FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, - ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, - double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, - ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, - ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, - int[] filterFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory) - throws HyracksDataException { + FileReference file, IBufferCache diskBufferCache, ITypeTraits[] typeTraits, + IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate, + ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, + ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits, + IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields, boolean durable, + IMetadataPageManagerFactory freePageManagerFactory) throws HyracksDataException { LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false); LSMBTreeTupleWriterFactory deleteTupleWriterFactory = @@ -82,15 +80,14 @@ public class LSMBTreeUtil { ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory); ITreeIndexFrameFactory bulkLoadLeafFrameFactory = new BTreeNSMLeafFrameFactory(bulkLoadTupleWriterFactory); - TreeIndexFactory<BTree> diskBTreeFactory = - new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, - interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length); + TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, + interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length); TreeIndexFactory<BTree> bulkLoadBTreeFactory = - new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, - interiorFrameFactory, bulkLoadLeafFrameFactory, cmpFactories, typeTraits.length); + new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, interiorFrameFactory, + bulkLoadLeafFrameFactory, cmpFactories, typeTraits.length); - BloomFilterFactory bloomFilterFactory = needKeyDupCheck - ? new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields) : null; + BloomFilterFactory bloomFilterFactory = + needKeyDupCheck ? new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields) : null; ComponentFilterHelper filterHelper = null; LSMComponentFilterFrameFactory filterFrameFactory = null; @@ -104,20 +101,20 @@ public class LSMBTreeUtil { //Primary LSMBTree index has a BloomFilter. ILSMIndexFileManager fileNameManager = - new LSMBTreeFileManager(ioManager, diskFileMapProvider, file, diskBTreeFactory, needKeyDupCheck); + new LSMBTreeFileManager(ioManager, file, diskBTreeFactory, needKeyDupCheck); return new LSMBTree(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory, - filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, diskFileMapProvider, - typeTraits.length, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck, - btreeFields, filterFields, durable); + filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, typeTraits.length, + cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck, btreeFields, + filterFields, durable); } public static ExternalBTree createExternalBTree(IIOManager ioManager, FileReference file, - IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, - IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate, - ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, - ILSMIOOperationCallback ioOpCallback, boolean durable, IMetadataPageManagerFactory freePageManagerFactory) { + IBufferCache diskBufferCache, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, + int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, + ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, + boolean durable, IMetadataPageManagerFactory freePageManagerFactory) { LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false); LSMBTreeTupleWriterFactory deleteTupleWriterFactory = @@ -136,35 +133,31 @@ public class LSMBTreeUtil { ITreeIndexFrameFactory transactionLeafFrameFactory = new BTreeNSMLeafFrameFactory(transactionTupleWriterFactory); - TreeIndexFactory<BTree> diskBTreeFactory = - new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, - interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length); - TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, + TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, + interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length); + TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length); - BloomFilterFactory bloomFilterFactory = - new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields); + BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields); // This is the component factory for transactions TreeIndexFactory<BTree> transactionBTreeFactory = - new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, - interiorFrameFactory, transactionLeafFrameFactory, cmpFactories, typeTraits.length); + new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, interiorFrameFactory, + transactionLeafFrameFactory, cmpFactories, typeTraits.length); //TODO remove BloomFilter from external dataset's secondary LSMBTree index - ILSMIndexFileManager fileNameManager = - new LSMBTreeFileManager(ioManager, diskFileMapProvider, file, diskBTreeFactory, true); + ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, file, diskBTreeFactory, true); // the disk only index uses an empty ArrayList for virtual buffer caches - ExternalBTree lsmTree = new ExternalBTree(ioManager, interiorFrameFactory, insertLeafFrameFactory, - deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory, - bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length, cmpFactories, mergePolicy, - opTracker, ioScheduler, ioOpCallback, transactionBTreeFactory, durable); - return lsmTree; + return new ExternalBTree(ioManager, interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, + fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory, + bloomFilterFalsePositiveRate, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, + transactionBTreeFactory, durable); } public static ExternalBTreeWithBuddy createExternalBTreeWithBuddy(IIOManager ioManager, FileReference file, - IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, - IBinaryComparatorFactory[] cmpFactories, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, - ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, - int[] buddyBTreeFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory) { + IBufferCache diskBufferCache, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, + double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, + ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields, + boolean durable, IMetadataPageManagerFactory freePageManagerFactory) { ITypeTraits[] buddyBtreeTypeTraits = new ITypeTraits[buddyBTreeFields.length]; IBinaryComparatorFactory[] buddyBtreeCmpFactories = new IBinaryComparatorFactory[buddyBTreeFields.length]; for (int i = 0; i < buddyBtreeTypeTraits.length; i++) { @@ -184,32 +177,30 @@ public class LSMBTreeUtil { ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory); ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory); ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory); - TreeIndexFactory<BTree> diskBTreeFactory = - new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, - interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length); + TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, + interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length); - TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, + TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length); int[] bloomFilterKeyFields = new int[buddyBtreeCmpFactories.length]; for (int i = 0; i < buddyBtreeCmpFactories.length; i++) { bloomFilterKeyFields[i] = i; } - BloomFilterFactory bloomFilterFactory = - new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields); + BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields); // buddy b-tree factory - TreeIndexFactory<BTree> diskBuddyBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, - diskFileMapProvider, freePageManagerFactory, buddyBtreeInteriorFrameFactory, buddyBtreeLeafFrameFactory, - buddyBtreeCmpFactories, buddyBtreeTypeTraits.length); + TreeIndexFactory<BTree> diskBuddyBTreeFactory = + new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, buddyBtreeInteriorFrameFactory, + buddyBtreeLeafFrameFactory, buddyBtreeCmpFactories, buddyBtreeTypeTraits.length); - ILSMIndexFileManager fileNameManager = new LSMBTreeWithBuddyFileManager(ioManager, diskFileMapProvider, file, - diskBTreeFactory, diskBuddyBTreeFactory); + ILSMIndexFileManager fileNameManager = + new LSMBTreeWithBuddyFileManager(ioManager, file, diskBTreeFactory, diskBuddyBTreeFactory); // the disk only index uses an empty ArrayList for virtual buffer caches ExternalBTreeWithBuddy lsmTree = new ExternalBTreeWithBuddy(ioManager, interiorFrameFactory, insertLeafFrameFactory, buddyBtreeLeafFrameFactory, diskBufferCache, fileNameManager, - bulkLoadBTreeFactory, diskBTreeFactory, diskBuddyBTreeFactory, bloomFilterFactory, diskFileMapProvider, + bulkLoadBTreeFactory, diskBTreeFactory, diskBuddyBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, cmpFactories, buddyBtreeCmpFactories, buddyBTreeFields, durable); return lsmTree; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml index bc1e146..37e1c49 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml @@ -82,8 +82,9 @@ <artifactId>commons-lang3</artifactId> </dependency> <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> + <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-util</artifactId> + <version>${project.version}</version> </dependency> </dependencies> </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java index 54d64af..18d9a67 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java @@ -34,28 +34,28 @@ import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences * merge). */ public interface ILSMIndexFileManager { - public void createDirs(); + void createDirs() throws HyracksDataException; - public void deleteDirs() throws HyracksDataException; + void deleteDirs() throws HyracksDataException; - public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException; + LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException; - public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName) + LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName) throws HyracksDataException; - public String getBaseDir(); + String getBaseDir(); // Deletes invalid files, and returns list of valid files from baseDir. // The returned valid files are correctly sorted (based on the recency of data). - public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException; + List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException; - public Comparator<String> getFileNameComparator(); + Comparator<String> getFileNameComparator(); /** * @return delete existing transaction disk component file reference * @throws HyracksDataException */ - public void deleteTransactionFiles() throws HyracksDataException; + void deleteTransactionFiles() throws HyracksDataException; /** * Rename files of a transaction removing the transaction prefix and return the component file reference in order to be committed @@ -63,18 +63,18 @@ public interface ILSMIndexFileManager { * @return the renamed component file references * @throws HyracksDataException */ - public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException; + LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException; /** * Recover transaction files without returning them * * @throws HyracksDataException */ - public void recoverTransaction() throws HyracksDataException; + void recoverTransaction() throws HyracksDataException; /** * @return a reference to the transaction disk component file reference * @throws IOException */ - public LSMComponentFileReferences getNewTransactionFileReference() throws IOException; + LSMComponentFileReferences getNewTransactionFileReference() throws IOException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java index 090ca68..72d5423 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java @@ -23,6 +23,6 @@ import org.apache.hyracks.api.io.IIOManager; public class AbortRecoverLSMIndexFileManager extends AbstractLSMIndexFileManager { public AbortRecoverLSMIndexFileManager(IIOManager ioManager, FileReference file) { - super(ioManager, null, file, null); + super(ioManager, file, null); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ae3daf6e/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java index 2be2184..c5bf2ca 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java @@ -63,7 +63,6 @@ import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.file.IFileMapProvider; public abstract class AbstractLSMIndex implements ILSMIndex { protected final ILSMHarness lsmHarness; @@ -75,11 +74,9 @@ public abstract class AbstractLSMIndex implements ILSMIndex { protected final List<ILSMMemoryComponent> memoryComponents; protected final List<IVirtualBufferCache> virtualBufferCaches; protected AtomicInteger currentMutableComponentId; - // On-disk components. protected final IBufferCache diskBufferCache; protected final ILSMIndexFileManager fileManager; - protected final IFileMapProvider diskFileMapProvider; // components with lower indexes are newer than components with higher index protected final List<ILSMDiskComponent> diskComponents; protected final List<ILSMDiskComponent> inactiveDiskComponents; @@ -95,16 +92,14 @@ public abstract class AbstractLSMIndex implements ILSMIndex { protected boolean memoryComponentsAllocated = false; public AbstractLSMIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, - IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, - double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, - ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, - ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, - int[] filterFields, boolean durable, IComponentFilterHelper filterHelper, - int[] treeFields) { + IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, double bloomFilterFalsePositiveRate, + ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, + ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFrameFactory filterFrameFactory, + LSMComponentFilterManager filterManager, int[] filterFields, boolean durable, + IComponentFilterHelper filterHelper, int[] treeFields) { this.ioManager = ioManager; this.virtualBufferCaches = virtualBufferCaches; this.diskBufferCache = diskBufferCache; - this.diskFileMapProvider = diskFileMapProvider; this.fileManager = fileManager; this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate; this.ioScheduler = ioScheduler; @@ -130,12 +125,10 @@ public abstract class AbstractLSMIndex implements ILSMIndex { // The constructor used by external indexes public AbstractLSMIndex(IIOManager ioManager, IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, - IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, - ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, - boolean durable) { + double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, + ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean durable) { this.ioManager = ioManager; this.diskBufferCache = diskBufferCache; - this.diskFileMapProvider = diskFileMapProvider; this.fileManager = fileManager; this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate; this.ioScheduler = ioScheduler; @@ -162,7 +155,6 @@ public abstract class AbstractLSMIndex implements ILSMIndex { if (isActive) { throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_ACTIVE_INDEX); } - fileManager.deleteDirs(); fileManager.createDirs(); diskComponents.clear(); } @@ -238,16 +230,9 @@ public abstract class AbstractLSMIndex implements ILSMIndex { throw HyracksDataException.create(ErrorCode.CANNOT_DESTROY_ACTIVE_INDEX); } destroyDiskComponents(); - destroyMemoryComponents(); fileManager.deleteDirs(); } - protected void destroyMemoryComponents() throws HyracksDataException { - for (ILSMMemoryComponent c : memoryComponents) { - destroyMemoryComponent(c); - } - } - protected void destroyDiskComponents() throws HyracksDataException { List<ILSMDiskComponent> immutableComponents = diskComponents; for (ILSMDiskComponent c : immutableComponents) { @@ -653,8 +638,6 @@ public abstract class AbstractLSMIndex implements ILSMIndex { protected abstract void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException; - protected abstract void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException; - protected abstract void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException; protected abstract void clearMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException;
