http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java index 4884190..70a5024 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java @@ -117,8 +117,7 @@ public class OnDiskInvertedIndex implements IInvertedIndex { IInvertedListBuilder invListBuilder, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference btreeFile, FileReference invListsFile, - IPageManagerFactory pageManagerFactory) - throws IndexException { + IPageManagerFactory pageManagerFactory) throws HyracksDataException { this.bufferCache = bufferCache; this.fileMapProvider = fileMapProvider; this.invListBuilder = invListBuilder; @@ -127,8 +126,8 @@ public class OnDiskInvertedIndex implements IInvertedIndex { this.tokenTypeTraits = tokenTypeTraits; this.tokenCmpFactories = tokenCmpFactories; this.btree = BTreeUtils.createBTree(bufferCache, fileMapProvider, getBTreeTypeTraits(tokenTypeTraits), - tokenCmpFactories, BTreeLeafFrameType.REGULAR_NSM, btreeFile, pageManagerFactory.createPageManager( - bufferCache)); + tokenCmpFactories, BTreeLeafFrameType.REGULAR_NSM, btreeFile, + pageManagerFactory.createPageManager(bufferCache)); this.numTokenFields = btree.getComparatorFactories().length; this.numInvListKeys = invListCmpFactories.length; this.invListsFile = invListsFile; @@ -280,14 +279,14 @@ public class OnDiskInvertedIndex implements IInvertedIndex { } public void resetInvertedListCursor(ITupleReference btreeTuple, IInvertedListCursor listCursor) { - int startPageId = IntegerPointable.getInteger(btreeTuple.getFieldData(invListStartPageIdField), btreeTuple - .getFieldStart(invListStartPageIdField)); - int endPageId = IntegerPointable.getInteger(btreeTuple.getFieldData(invListEndPageIdField), btreeTuple - .getFieldStart(invListEndPageIdField)); - int startOff = IntegerPointable.getInteger(btreeTuple.getFieldData(invListStartOffField), btreeTuple - .getFieldStart(invListStartOffField)); - int numElements = IntegerPointable.getInteger(btreeTuple.getFieldData(invListNumElementsField), btreeTuple - .getFieldStart(invListNumElementsField)); + int startPageId = IntegerPointable.getInteger(btreeTuple.getFieldData(invListStartPageIdField), + btreeTuple.getFieldStart(invListStartPageIdField)); + int endPageId = IntegerPointable.getInteger(btreeTuple.getFieldData(invListEndPageIdField), + btreeTuple.getFieldStart(invListEndPageIdField)); + int startOff = IntegerPointable.getInteger(btreeTuple.getFieldData(invListStartOffField), + btreeTuple.getFieldStart(invListStartOffField)); + int numElements = IntegerPointable.getInteger(btreeTuple.getFieldData(invListNumElementsField), + btreeTuple.getFieldStart(invListNumElementsField)); listCursor.reset(startPageId, endPageId, startOff, numElements); } @@ -312,8 +311,7 @@ public class OnDiskInvertedIndex implements IInvertedIndex { private IFIFOPageQueue queue; public OnDiskInvertedIndexBulkLoader(float btreeFillFactor, boolean verifyInput, long numElementsHint, - boolean checkIfEmptyIndex, int startPageId) throws IndexException, - HyracksDataException { + boolean checkIfEmptyIndex, int startPageId) throws IndexException, HyracksDataException { this.verifyInput = verifyInput; this.tokenCmp = MultiComparator.create(btree.getComparatorFactories()); this.invListCmp = MultiComparator.create(invListCmpFactories); @@ -326,8 +324,8 @@ public class OnDiskInvertedIndex implements IInvertedIndex { this.btreeTupleReference = new ArrayTupleReference(); this.lastTupleBuilder = new ArrayTupleBuilder(numTokenFields + numInvListKeys); this.lastTuple = new ArrayTupleReference(); - this.btreeBulkloader = btree.createBulkLoader(btreeFillFactor, verifyInput, numElementsHint, - checkIfEmptyIndex); + this.btreeBulkloader = + btree.createBulkLoader(btreeFillFactor, verifyInput, numElementsHint, checkIfEmptyIndex); currentPageId = startPageId; currentPage = bufferCache.confiscatePage(BufferedFileHandle.getDiskPageId(fileId, currentPageId)); invListBuilder.setTargetBuffer(currentPage.getBuffer().array(), 0); @@ -507,9 +505,10 @@ public class OnDiskInvertedIndex implements IInvertedIndex { } @Override - public void search(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException, - IndexException { - searcher.search((OnDiskInvertedIndexSearchCursor) cursor, (InvertedIndexSearchPredicate) searchPred, opCtx); + public void search(IIndexCursor cursor, ISearchPredicate searchPred) + throws HyracksDataException, IndexException { + searcher.search((OnDiskInvertedIndexSearchCursor) cursor, (InvertedIndexSearchPredicate) searchPred, + opCtx); } @Override @@ -529,8 +528,8 @@ public class OnDiskInvertedIndex implements IInvertedIndex { } @Override - public void rangeSearch(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException, - IndexException { + public void rangeSearch(IIndexCursor cursor, ISearchPredicate searchPred) + throws HyracksDataException, IndexException { OnDiskInvertedIndexRangeSearchCursor rangeSearchCursor = (OnDiskInvertedIndexRangeSearchCursor) cursor; rangeSearchCursor.open(null, searchPred); } @@ -617,8 +616,8 @@ public class OnDiskInvertedIndex implements IInvertedIndex { public void validate() throws HyracksDataException { btree.validate(); // Scan the btree and validate the order of elements in each inverted-list. - IIndexAccessor btreeAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE, - NoOpOperationCallback.INSTANCE); + IIndexAccessor btreeAccessor = + btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); IIndexCursor btreeCursor = btreeAccessor.createSearchCursor(false); MultiComparator btreeCmp = MultiComparator.create(btree.getComparatorFactories()); RangePredicate rangePred = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp); @@ -628,8 +627,9 @@ public class OnDiskInvertedIndex implements IInvertedIndex { } PermutingTupleReference tokenTuple = new PermutingTupleReference(fieldPermutation); - IInvertedIndexAccessor invIndexAccessor = (IInvertedIndexAccessor) createAccessor( - NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); + IInvertedIndexAccessor invIndexAccessor = + (IInvertedIndexAccessor) createAccessor(NoOpOperationCallback.INSTANCE, + NoOpOperationCallback.INSTANCE); IInvertedListCursor invListCursor = invIndexAccessor.createInvertedListCursor(); MultiComparator invListCmp = MultiComparator.create(invListCmpFactories); @@ -706,5 +706,4 @@ public class OnDiskInvertedIndex implements IInvertedIndex { public boolean hasMemoryComponents() { return true; } - }
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java index 14ceee8..dacef3c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java @@ -24,7 +24,6 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; -import org.apache.hyracks.storage.am.common.api.IndexException; import org.apache.hyracks.storage.am.lsm.common.impls.IndexFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNameMapper; @@ -57,7 +56,7 @@ public class OnDiskInvertedIndexFactory extends IndexFactory<IInvertedIndex> { } @Override - public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws IndexException, HyracksDataException { + public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws HyracksDataException { String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getAbsolutePath()); FileReference invListsFile = ioManager.resolveAbsolutePath(invListsFilePath); IInvertedListBuilder invListBuilder = invListBuilderFactory.create(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java index 6d47ddc..a3a4de4 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java @@ -50,8 +50,7 @@ public class PartitionedOnDiskInvertedIndex extends OnDiskInvertedIndex implemen IInvertedListBuilder invListBuilder, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference btreeFile, FileReference invListsFile, - IPageManagerFactory pageManagerFactory) - throws IndexException { + IPageManagerFactory pageManagerFactory) throws HyracksDataException { super(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory); } @@ -99,7 +98,8 @@ public class PartitionedOnDiskInvertedIndex extends OnDiskInvertedIndex implemen while (ctx.btreeCursor.hasNext()) { ctx.btreeCursor.next(); ITupleReference btreeTuple = ctx.btreeCursor.getTuple(); - short numTokens = ShortPointable.getShort(btreeTuple.getFieldData(PARTITIONING_NUM_TOKENS_FIELD), btreeTuple.getFieldStart(PARTITIONING_NUM_TOKENS_FIELD)); + short numTokens = ShortPointable.getShort(btreeTuple.getFieldData(PARTITIONING_NUM_TOKENS_FIELD), + btreeTuple.getFieldStart(PARTITIONING_NUM_TOKENS_FIELD)); IInvertedListCursor invListCursor = partSearcher.getCachedInvertedListCursor(); resetInvertedListCursor(btreeTuple, invListCursor); cursorsOrderedByTokens.add(invListCursor); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java index b97cad0..ebc0152 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java @@ -24,7 +24,6 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; -import org.apache.hyracks.storage.am.common.api.IndexException; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNameMapper; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder; @@ -35,17 +34,16 @@ import org.apache.hyracks.storage.common.file.IFileMapProvider; public class PartitionedOnDiskInvertedIndexFactory extends OnDiskInvertedIndexFactory { public PartitionedOnDiskInvertedIndexFactory(IIOManager ioManager, IBufferCache bufferCache, - IFileMapProvider fileMapProvider, - IInvertedListBuilderFactory invListBuilderFactory, ITypeTraits[] invListTypeTraits, - IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, - IBinaryComparatorFactory[] tokenCmpFactories, IInvertedIndexFileNameMapper fileNameMapper, - IPageManagerFactory pageManagerFactory) { + IFileMapProvider fileMapProvider, IInvertedListBuilderFactory invListBuilderFactory, + ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, + ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, + IInvertedIndexFileNameMapper fileNameMapper, IPageManagerFactory pageManagerFactory) { super(ioManager, bufferCache, fileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, fileNameMapper, pageManagerFactory); } @Override - public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws IndexException, HyracksDataException { + public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws HyracksDataException { String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getAbsolutePath()); FileReference invListsFile = ioManager.resolveAbsolutePath(invListsFilePath); IInvertedListBuilder invListBuilder = invListBuilderFactory.create(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java index b8dd258..c6a552c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java @@ -27,7 +27,6 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory; -import org.apache.hyracks.storage.am.btree.exceptions.BTreeException; import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType; import org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory; import org.apache.hyracks.storage.am.btree.util.BTreeUtils; @@ -69,7 +68,7 @@ public class InvertedIndexUtils { IPageManager virtualFreePageManager, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, - FileReference btreeFileRef) throws BTreeException { + FileReference btreeFileRef) throws HyracksDataException { return new InMemoryInvertedIndex(memBufferCache, virtualFreePageManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, btreeFileRef); } @@ -78,7 +77,7 @@ public class InvertedIndexUtils { IPageManager virtualFreePageManager, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, - FileReference btreeFileRef) throws BTreeException { + FileReference btreeFileRef) throws HyracksDataException { return new PartitionedInMemoryInvertedIndex(memBufferCache, virtualFreePageManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, btreeFileRef); } @@ -87,8 +86,7 @@ public class InvertedIndexUtils { IFileMapProvider fileMapProvider, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile, - IPageManagerFactory pageManagerFactory) throws IndexException, - HyracksDataException { + IPageManagerFactory pageManagerFactory) throws IndexException, HyracksDataException { IInvertedListBuilder builder = new FixedSizeElementInvertedListBuilder(invListTypeTraits); FileReference btreeFile = getBTreeFile(ioManager, invListsFile); return new OnDiskInvertedIndex(bufferCache, fileMapProvider, builder, invListTypeTraits, invListCmpFactories, @@ -96,12 +94,10 @@ public class InvertedIndexUtils { } public static PartitionedOnDiskInvertedIndex createPartitionedOnDiskInvertedIndex(IIOManager ioManager, - IBufferCache bufferCache, - IFileMapProvider fileMapProvider, ITypeTraits[] invListTypeTraits, + IBufferCache bufferCache, IFileMapProvider fileMapProvider, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile, - IPageManagerFactory pageManagerFactory) throws IndexException, - HyracksDataException { + IPageManagerFactory pageManagerFactory) throws IndexException, HyracksDataException { IInvertedListBuilder builder = new FixedSizeElementInvertedListBuilder(invListTypeTraits); FileReference btreeFile = getBTreeFile(ioManager, invListsFile); return new PartitionedOnDiskInvertedIndex(bufferCache, fileMapProvider, builder, invListTypeTraits, @@ -113,30 +109,30 @@ public class InvertedIndexUtils { return ioManager.resolveAbsolutePath(invListsFile.getFile().getPath() + "_btree"); } - public static BTreeFactory createDeletedKeysBTreeFactory(IIOManager ioManager, IFileMapProvider diskFileMapProvider, - ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, - IBufferCache diskBufferCache, IPageManagerFactory freePageManagerFactory) throws BTreeException { + public static BTreeFactory createDeletedKeysBTreeFactory(IIOManager ioManager, + IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits, + IBinaryComparatorFactory[] invListCmpFactories, IBufferCache diskBufferCache, + IPageManagerFactory freePageManagerFactory) throws HyracksDataException { TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(invListTypeTraits); - ITreeIndexFrameFactory leafFrameFactory = BTreeUtils.getLeafFrameFactory(tupleWriterFactory, - BTreeLeafFrameType.REGULAR_NSM); + ITreeIndexFrameFactory leafFrameFactory = + BTreeUtils.getLeafFrameFactory(tupleWriterFactory, BTreeLeafFrameType.REGULAR_NSM); ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory); - BTreeFactory deletedKeysBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, - freePageManagerFactory, interiorFrameFactory, leafFrameFactory, invListCmpFactories, - invListCmpFactories.length); + BTreeFactory deletedKeysBTreeFactory = + new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, + interiorFrameFactory, leafFrameFactory, invListCmpFactories, invListCmpFactories.length); return deletedKeysBTreeFactory; } public static LSMInvertedIndex createLSMInvertedIndex(IIOManager ioManager, - List<IVirtualBufferCache> virtualBufferCaches, - IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits, - IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, - IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, - IBufferCache diskBufferCache, String absoluteOnDiskDir, double bloomFilterFalsePositiveRate, - ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, - ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, - IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, - int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, IMetadataPageManagerFactory pageManagerFactory) - throws IndexException, HyracksDataException { + List<IVirtualBufferCache> virtualBufferCaches, IFileMapProvider diskFileMapProvider, + ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, + ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, + IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String absoluteOnDiskDir, + double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, + ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, + ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, + int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, + IMetadataPageManagerFactory pageManagerFactory) throws IndexException, HyracksDataException { BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory); @@ -145,15 +141,15 @@ public class InvertedIndexUtils { for (int i = 0; i < invListCmpFactories.length; i++) { bloomFilterKeyFields[i] = i; } - BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider, - bloomFilterKeyFields); + BloomFilterFactory bloomFilterFactory = + new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields); FileReference onDiskDirFileRef = ioManager.resolveAbsolutePath(absoluteOnDiskDir); LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider, onDiskDirFileRef, deletedKeysBTreeFactory); - IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory( - invListTypeTraits); + IInvertedListBuilderFactory invListBuilderFactory = + new FixedSizeElementInvertedListBuilderFactory(invListTypeTraits); OnDiskInvertedIndexFactory invIndexFactory = new OnDiskInvertedIndexFactory(ioManager, diskBufferCache, diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, fileManager, pageManagerFactory); @@ -168,12 +164,11 @@ public class InvertedIndexUtils { filterManager = new LSMComponentFilterManager(filterFrameFactory); } LSMInvertedIndex invIndex = new LSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory, - deletedKeysBTreeFactory, - bloomFilterFactory, filterFactory, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, - fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, - tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, - invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, - durable); + deletedKeysBTreeFactory, bloomFilterFactory, filterFactory, filterFrameFactory, filterManager, + bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, + tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, + ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, + invertedIndexFieldsForNonBulkLoadOps, durable); return invIndex; } @@ -186,8 +181,7 @@ public class InvertedIndexUtils { ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, - IPageManagerFactory pageManagerFactory) - throws IndexException, HyracksDataException { + IPageManagerFactory pageManagerFactory) throws IndexException, HyracksDataException { BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory); @@ -196,15 +190,15 @@ public class InvertedIndexUtils { for (int i = 0; i < invListCmpFactories.length; i++) { bloomFilterKeyFields[i] = i; } - BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider, - bloomFilterKeyFields); + BloomFilterFactory bloomFilterFactory = + new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields); FileReference onDiskDirFileRef = ioManager.resolveAbsolutePath(absoluteOnDiskDir); LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider, onDiskDirFileRef, deletedKeysBTreeFactory); - IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory( - invListTypeTraits); + IInvertedListBuilderFactory invListBuilderFactory = + new FixedSizeElementInvertedListBuilderFactory(invListTypeTraits); PartitionedOnDiskInvertedIndexFactory invIndexFactory = new PartitionedOnDiskInvertedIndexFactory(ioManager, diskBufferCache, diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, fileManager, pageManagerFactory); @@ -219,11 +213,10 @@ public class InvertedIndexUtils { filterManager = new LSMComponentFilterManager(filterFrameFactory); } PartitionedLSMInvertedIndex invIndex = new PartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, - invIndexFactory, - deletedKeysBTreeFactory, bloomFilterFactory, filterFactory, filterFrameFactory, filterManager, - bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, - tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, - ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, + invIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterFactory, filterFrameFactory, + filterManager, bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits, + invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, + ioScheduler, ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable); return invIndex; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java index 7c57bed..1b2ff6f 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java @@ -100,7 +100,7 @@ public class ExternalRTreeDataflowHelper extends LSMRTreeDataflowHelper { typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, - ioOpCallbackFactory.createIOOperationCallback(), linearizeCmpFactory, btreeFields, version, durable, + ioOpCallbackFactory.createIoOpCallback(), linearizeCmpFactory, btreeFields, version, durable, isPointMBR, (IMetadataPageManagerFactory) opDesc.getPageManagerFactory()); } catch (TreeIndexException e) { throw new HyracksDataException(e); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java index 3f69016..63633aa 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java @@ -89,7 +89,7 @@ public class LSMRTreeDataflowHelper extends AbstractLSMRTreeDataflowHelper { diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, - ioOpCallbackFactory.createIOOperationCallback(), linearizeCmpFactory, rtreeFields, btreeFields, + ioOpCallbackFactory.createIoOpCallback(), linearizeCmpFactory, rtreeFields, btreeFields, filterTypeTraits, filterCmpFactories, filterFields, durable, isPointMBR, (IMetadataPageManagerFactory) opDesc.getPageManagerFactory()); } catch (TreeIndexException e) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java index 8a807c2..6e0ffaf 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java @@ -70,7 +70,7 @@ public class LSMRTreeWithAntiMatterTuplesDataflowHelper extends AbstractLSMRTree return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(ctx.getIOManager(), virtualBufferCaches, file, diskBufferCache, diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory - .createIOOperationCallback(), linearizeCmpFactory, rtreeFields, filterTypeTraits, + .createIoOpCallback(), linearizeCmpFactory, rtreeFields, filterTypeTraits, filterCmpFactories, filterFields, durable, isPointMBR, (IMetadataPageManagerFactory) opDesc .getPageManagerFactory()); } catch (TreeIndexException e) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java index fc5b6b8..480e1e2 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java @@ -41,8 +41,9 @@ import org.apache.hyracks.storage.am.common.exceptions.TreeIndexDuplicateKeyExce import org.apache.hyracks.storage.am.common.impls.AbstractSearchPredicate; import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; @@ -71,7 +72,7 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree // On-disk components. // For creating RTree's used in flush and merge. - protected final ILSMComponentFactory componentFactory; + protected final ILSMDiskComponentFactory componentFactory; protected IBinaryComparatorFactory[] btreeCmpFactories; protected IBinaryComparatorFactory[] rtreeCmpFactories; @@ -86,30 +87,31 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree public AbstractLSMRTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory, - ILSMIndexFileManager fileManager, ILSMComponentFactory componentFactory, + ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory, IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFactory filterFactory, ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, - int[] rtreeFields, int[] filterFields, boolean durable, boolean isPointMBR) throws HyracksDataException { - super(ioManager, virtualBufferCaches, componentFactory.getBufferCache(), fileManager, diskFileMapProvider, + int[] rtreeFields, int[] filterFields, boolean durable, boolean isPointMBR, IBufferCache diskBufferCache) + throws HyracksDataException { + super(ioManager, virtualBufferCaches, diskBufferCache, fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory, filterManager, filterFields, durable); int i = 0; for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) { RTree memRTree = new RTree(virtualBufferCache, virtualBufferCache.getFileMapProvider(), - new VirtualFreePageManager(virtualBufferCache), rtreeInteriorFrameFactory, - rtreeLeafFrameFactory, rtreeCmpFactories, fieldCount, + new VirtualFreePageManager(virtualBufferCache), rtreeInteriorFrameFactory, rtreeLeafFrameFactory, + rtreeCmpFactories, fieldCount, ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_r_" + i), isPointMBR); BTree memBTree = new BTree(virtualBufferCache, virtualBufferCache.getFileMapProvider(), - new VirtualFreePageManager(virtualBufferCache), btreeInteriorFrameFactory, - btreeLeafFrameFactory, btreeCmpFactories, btreeCmpFactories.length, + new VirtualFreePageManager(virtualBufferCache), btreeInteriorFrameFactory, btreeLeafFrameFactory, + btreeCmpFactories, btreeCmpFactories.length, ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_b_" + i)); - LSMRTreeMemoryComponent mutableComponent = new LSMRTreeMemoryComponent(memRTree, memBTree, - virtualBufferCache, i == 0 ? true : false, - filterFactory == null ? null : filterFactory.createLSMComponentFilter()); + LSMRTreeMemoryComponent mutableComponent = + new LSMRTreeMemoryComponent(memRTree, memBTree, virtualBufferCache, i == 0 ? true : false, + filterFactory == null ? null : filterFactory.createFilter()); memoryComponents.add(mutableComponent); ++i; } @@ -134,15 +136,14 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree public AbstractLSMRTree(IIOManager ioManager, ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory, ILSMIndexFileManager fileManager, - ILSMComponentFactory componentFactory, IFileMapProvider diskFileMapProvider, int fieldCount, + ILSMDiskComponentFactory componentFactory, IFileMapProvider diskFileMapProvider, IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean durable, - boolean isPointMBR) { - super(ioManager, componentFactory.getBufferCache(), fileManager, diskFileMapProvider, - bloomFilterFalsePositiveRate, - mergePolicy, opTracker, ioScheduler, ioOpCallback, durable); + boolean isPointMBR, IBufferCache diskBufferCache) { + super(ioManager, diskBufferCache, fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy, + opTracker, ioScheduler, ioOpCallback, durable); this.rtreeInteriorFrameFactory = rtreeInteriorFrameFactory; this.rtreeLeafFrameFactory = rtreeLeafFrameFactory; this.btreeInteriorFrameFactory = btreeInteriorFrameFactory; @@ -183,7 +184,8 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree if (flushOnExit) { BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback); - ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); + ILSMIndexAccessor accessor = + createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); accessor.scheduleFlush(cb); try { cb.waitForIO(); @@ -212,7 +214,7 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree @Override public void getOperationalComponents(ILSMIndexOperationContext ctx) throws HyracksDataException { List<ILSMComponent> operationalComponents = ctx.getComponentHolder(); - List<ILSMComponent> immutableComponents = diskComponents; + List<ILSMDiskComponent> immutableComponents = diskComponents; int cmc = currentMutableComponentId.get(); ctx.setCurrentMutableComponentId(cmc); operationalComponents.clear(); @@ -264,21 +266,21 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree protected LSMComponentFileReferences getMergeTargetFileName(List<ILSMComponent> mergingDiskComponents) throws HyracksDataException { RTree firstTree = ((LSMRTreeDiskComponent) mergingDiskComponents.get(0)).getRTree(); - RTree lastTree = ((LSMRTreeDiskComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1)) - .getRTree(); + RTree lastTree = + ((LSMRTreeDiskComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1)).getRTree(); FileReference firstFile = firstTree.getFileReference(); FileReference lastFile = lastTree.getFileReference(); - LSMComponentFileReferences fileRefs = fileManager.getRelMergeFileReference(firstFile.getFile().getName(), - lastFile.getFile().getName()); + LSMComponentFileReferences fileRefs = + fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName()); return fileRefs; } - protected LSMRTreeDiskComponent createDiskComponent(ILSMComponentFactory factory, FileReference insertFileRef, + protected LSMRTreeDiskComponent createDiskComponent(ILSMDiskComponentFactory factory, FileReference insertFileRef, FileReference deleteFileRef, FileReference bloomFilterFileRef, boolean createComponent) throws HyracksDataException, IndexException { // Create new tree instance. - LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) factory.createLSMComponentInstance( - new LSMComponentFileReferences(insertFileRef, deleteFileRef, bloomFilterFileRef)); + LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) factory + .createComponent(new LSMComponentFileReferences(insertFileRef, deleteFileRef, bloomFilterFileRef)); // Tree will be closed during cleanup of merge(). if (createComponent) { component.getRTree().create(); @@ -293,55 +295,56 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree component.getBloomFilter().activate(); } if (component.getLSMComponentFilter() != null && !createComponent) { - filterManager.readFilterInfo(component.getLSMComponentFilter(), component.getRTree()); + filterManager.readFilter(component.getLSMComponentFilter(), component.getRTree()); } return component; } @Override public ITreeIndexFrameFactory getLeafFrameFactory() { - LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) memoryComponents - .get(currentMutableComponentId.get()); + LSMRTreeMemoryComponent mutableComponent = + (LSMRTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get()); return mutableComponent.getRTree().getLeafFrameFactory(); } @Override public ITreeIndexFrameFactory getInteriorFrameFactory() { - LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) memoryComponents - .get(currentMutableComponentId.get()); + LSMRTreeMemoryComponent mutableComponent = + (LSMRTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get()); return mutableComponent.getRTree().getInteriorFrameFactory(); } @Override public IPageManager getPageManager() { - LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) memoryComponents - .get(currentMutableComponentId.get()); + LSMRTreeMemoryComponent mutableComponent = + (LSMRTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get()); return mutableComponent.getRTree().getPageManager(); } @Override public int getFieldCount() { - LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) memoryComponents - .get(currentMutableComponentId.get()); + LSMRTreeMemoryComponent mutableComponent = + (LSMRTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get()); return mutableComponent.getRTree().getFieldCount(); } @Override public int getRootPageId() { - LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) memoryComponents - .get(currentMutableComponentId.get()); + LSMRTreeMemoryComponent mutableComponent = + (LSMRTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get()); return mutableComponent.getRTree().getRootPageId(); } @Override public int getFileId() { - LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) memoryComponents - .get(currentMutableComponentId.get()); + LSMRTreeMemoryComponent mutableComponent = + (LSMRTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get()); return mutableComponent.getRTree().getFileId(); } @Override - public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException { + public void modify(IIndexOperationContext ictx, ITupleReference tuple) + throws HyracksDataException, IndexException { LSMRTreeOpContext ctx = (LSMRTreeOpContext) ictx; if (ctx.getOperation() == IndexOperation.PHYSICALDELETE) { throw new UnsupportedOperationException("Physical delete not supported in the LSM-RTree"); @@ -380,8 +383,8 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree protected LSMRTreeOpContext createOpContext(IModificationOperationCallback modCallback, ISearchOperationCallback searchCallback) { return new LSMRTreeOpContext(memoryComponents, rtreeLeafFrameFactory, rtreeInteriorFrameFactory, - btreeLeafFrameFactory, btreeInteriorFrameFactory, rtreeCmpFactories, btreeCmpFactories, modCallback, - searchCallback, rtreeFields, filterFields, lsmHarness, comparatorFields, linearizerArray); + btreeLeafFrameFactory, modCallback, searchCallback, rtreeFields, filterFields, lsmHarness, + comparatorFields, linearizerArray); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java index bfaba68..f846c6c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java @@ -46,11 +46,12 @@ import org.apache.hyracks.storage.am.common.api.IndexException; import org.apache.hyracks.storage.am.common.api.TreeIndexException; import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy; @@ -76,7 +77,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { // A second disk component list that will be used when a transaction is // committed and will be seen by subsequent accessors - private final List<ILSMComponent> secondDiskComponents; + private final List<ILSMDiskComponent> secondDiskComponents; // A pointer that points to the current most recent list (either // diskComponents = 0, or secondDiskComponents = 1). It starts with -1 to // indicate first time activation @@ -84,49 +85,34 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { private final int fieldCount; public ExternalRTree(IIOManager ioManager, ITreeIndexFrameFactory rtreeInteriorFrameFactory, - ITreeIndexFrameFactory rtreeLeafFrameFactory, - ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory, - ILSMIndexFileManager fileNameManager, TreeIndexFactory<RTree> diskRTreeFactory, - TreeIndexFactory<BTree> diskBTreeFactory, BloomFilterFactory bloomFilterFactory, - double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount, - IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories, - ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, - ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, - ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields, int version, boolean durable, - boolean isPointMBR) { + ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory, + ITreeIndexFrameFactory btreeLeafFrameFactory, ILSMIndexFileManager fileNameManager, + TreeIndexFactory<RTree> diskRTreeFactory, TreeIndexFactory<BTree> diskBTreeFactory, + BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate, + IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories, + IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer, + int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMMergePolicy mergePolicy, + ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, + int[] buddyBTreeFields, int version, boolean durable, boolean isPointMBR) { super(ioManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, - btreeLeafFrameFactory, - fileNameManager, diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, - diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields, - linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, buddyBTreeFields, durable, - isPointMBR); + btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, + bloomFilterFalsePositiveRate, diskFileMapProvider, rtreeCmpFactories, btreeCmpFactories, linearizer, + comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, buddyBTreeFields, + durable, isPointMBR); this.secondDiskComponents = new LinkedList<>(); this.version = version; this.fieldCount = fieldCount; } - // This method is used to create a target for a bulk modify operation. This - // component must then eventually be either committed or deleted - private ILSMComponent createTransactionTarget() throws HyracksDataException, IndexException { - LSMComponentFileReferences componentFileRefs; - try { - componentFileRefs = fileManager.getNewTransactionFileReference(); - } catch (IOException e) { - throw new HyracksDataException("Failed to create transaction components", e); - } - return createDiskComponent(componentFactory, componentFileRefs.getInsertIndexFileReference(), - componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true); - } - // The subsume merged components is overridden to account for: // 1. the number of readers of components // 2. maintaining two versions of the index @Override - public void subsumeMergedComponents(ILSMComponent newComponent, List<ILSMComponent> mergedComponents) + public void subsumeMergedComponents(ILSMDiskComponent newComponent, List<ILSMComponent> mergedComponents) throws HyracksDataException { // determine which list is the new one - List<ILSMComponent> newerList; - List<ILSMComponent> olderList; + List<ILSMDiskComponent> newerList; + List<ILSMDiskComponent> olderList; if (version == 0) { newerList = diskComponents; olderList = secondDiskComponents; @@ -150,7 +136,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { // is needed. // It only needs to return the newer list @Override - public List<ILSMComponent> getImmutableComponents() { + public List<ILSMDiskComponent> getImmutableComponents() { if (version == 0) { return diskComponents; } else { @@ -169,7 +155,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { // is successful // it will therefore add the component to the first list and enter it. @Override - public void addComponent(ILSMComponent c) throws HyracksDataException { + public void addDiskComponent(ILSMDiskComponent c) throws HyracksDataException { if (version == 0) { diskComponents.add(0, c); } else if (version == 1) { @@ -179,11 +165,11 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { // This function is used when a new component is to be committed. @Override - public void commitTransactionDiskComponent(ILSMComponent newComponent) throws HyracksDataException { + public void commitTransactionDiskComponent(ILSMDiskComponent newComponent) throws HyracksDataException { // determine which list is the new one and flip the pointer - List<ILSMComponent> newerList; - List<ILSMComponent> olderList; + List<ILSMDiskComponent> newerList; + List<ILSMDiskComponent> olderList; if (version == 0) { newerList = diskComponents; olderList = secondDiskComponents; @@ -281,7 +267,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { // This can be done in a better way by creating a method boolean // keepDeletedTuples(mergedComponents); @Override - public ILSMComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException { + public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException { LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation; ITreeIndexCursor cursor = mergeOp.getCursor(); ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null); @@ -313,8 +299,8 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { } int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements); - BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement, - bloomFilterFalsePositiveRate); + BloomFilterSpecification bloomFilterSpec = + BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate); IIndexBulkLoader builder = mergedComponent.getBloomFilter().createBuilder(numElements, bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements()); @@ -444,7 +430,8 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { // Not supported @Override - public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException { + public void modify(IIndexOperationContext ictx, ITupleReference tuple) + throws HyracksDataException, IndexException { throw new UnsupportedOperationException("tuple modify not supported in LSM-Disk-Only-RTree"); } @@ -457,7 +444,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { // Not supported @Override - public ILSMComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException { + public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException { throw new UnsupportedOperationException("flush not supported in LSM-Disk-Only-RTree"); } @@ -465,7 +452,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { @Override public void getOperationalComponents(ILSMIndexOperationContext ctx) { List<ILSMComponent> operationalComponents = ctx.getComponentHolder(); - List<ILSMComponent> immutableComponents; + List<ILSMDiskComponent> immutableComponents; // Identify current list in case of a merge if (version == 0) { immutableComponents = diskComponents; @@ -527,7 +514,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { // The bulk loader used for both initial loading and transaction // modifications public class LSMTwoPCRTreeBulkLoader implements IIndexBulkLoader, ITwoPCIndexBulkLoader { - private final ILSMComponent component; + private final ILSMDiskComponent component; private final IIndexBulkLoader rtreeBulkLoader; private final BTreeBulkLoader btreeBulkLoader; private final IIndexBulkLoader builder; @@ -563,8 +550,8 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { btreeBulkLoader = (BTreeBulkLoader) ((LSMRTreeDiskComponent) component).getBTree() .createBulkLoader(fillFactor, verifyInput, numElementsHint, false); int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint); - BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement, - bloomFilterFalsePositiveRate); + BloomFilterSpecification bloomFilterSpec = + BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate); builder = ((LSMRTreeDiskComponent) component).getBloomFilter().createBuilder(numElementsHint, bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements()); } @@ -657,6 +644,20 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { } } + + // This method is used to create a target for a bulk modify operation. This + // component must then eventually be either committed or deleted + private ILSMDiskComponent createTransactionTarget() throws HyracksDataException, IndexException { + LSMComponentFileReferences componentFileRefs; + try { + componentFileRefs = fileManager.getNewTransactionFileReference(); + } catch (IOException e) { + throw new HyracksDataException("Failed to create transaction components", e); + } + return createDiskComponent(componentFactory, componentFileRefs.getInsertIndexFileReference(), + componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), + true); + } } @Override @@ -674,7 +675,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { List<ILSMComponent> mergingComponents = ctx.getComponentHolder(); ITreeIndexCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer, buddyBTreeFields); LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents); - ILSMIndexAccessorInternal accessor = new LSMRTreeAccessor(lsmHarness, rctx); + ILSMIndexAccessor accessor = new LSMRTreeAccessor(lsmHarness, rctx); // create the merge operation. LSMRTreeMergeOperation mergeOp = new LSMRTreeMergeOperation(accessor, mergingComponents, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(), @@ -695,7 +696,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { } @Override - public ILSMIndexAccessorInternal createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion) + public ILSMIndexAccessor createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion) throws HyracksDataException { return new LSMRTreeAccessor(lsmHarness, createOpContext(searchCallback, targetIndexVersion)); } @@ -710,7 +711,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { // The accessor for disk only indexes don't use modification callback and // always carry the target index version with them @Override - public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback, + public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) { return new LSMRTreeAccessor(lsmHarness, createOpContext(searchCallback, version)); } @@ -721,12 +722,12 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { } @Override - public List<ILSMComponent> getFirstComponentList() { + public List<ILSMDiskComponent> getFirstComponentList() { return diskComponents; } @Override - public List<ILSMComponent> getSecondComponentList() { + public List<ILSMDiskComponent> getSecondComponentList() { return secondDiskComponents; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java index 358a42a..dcd2c67 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java @@ -28,6 +28,7 @@ import org.apache.hyracks.storage.am.common.api.ISearchPredicate; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext; @@ -37,8 +38,8 @@ public class ExternalRTreeOpContext extends AbstractLSMIndexOperationContext { private MultiComparator bTreeCmp; private MultiComparator rTreeCmp; public final List<ILSMComponent> componentHolder; - private final List<ILSMComponent> componentsToBeMerged; - private final List<ILSMComponent> componentsToBeReplicated; + private final List<ILSMDiskComponent> componentsToBeMerged; + private final List<ILSMDiskComponent> componentsToBeReplicated; public final ISearchOperationCallback searchCallback; private final int targetIndexVersion; public ISearchPredicate searchPredicate; @@ -49,16 +50,16 @@ public class ExternalRTreeOpContext extends AbstractLSMIndexOperationContext { int targetIndexVersion, ILSMHarness lsmHarness, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory) { - this.componentHolder = new LinkedList<ILSMComponent>(); - this.componentsToBeMerged = new LinkedList<ILSMComponent>(); - this.componentsToBeReplicated = new LinkedList<ILSMComponent>(); + this.componentHolder = new LinkedList<>(); + this.componentsToBeMerged = new LinkedList<>(); + this.componentsToBeReplicated = new LinkedList<>(); this.searchCallback = searchCallback; this.targetIndexVersion = targetIndexVersion; this.bTreeCmp = MultiComparator.create(btreeCmpFactories); this.rTreeCmp = MultiComparator.create(rtreeCmpFactories); - initialState = new LSMRTreeCursorInitialState(rtreeLeafFrameFactory, rtreeInteriorFrameFactory, - btreeLeafFrameFactory, bTreeCmp, lsmHarness, comparatorFields, linearizerArray, searchCallback, - componentHolder); + initialState = + new LSMRTreeCursorInitialState(rtreeLeafFrameFactory, rtreeInteriorFrameFactory, btreeLeafFrameFactory, + bTreeCmp, lsmHarness, comparatorFields, linearizerArray, searchCallback, componentHolder); } @Override @@ -110,7 +111,7 @@ public class ExternalRTreeOpContext extends AbstractLSMIndexOperationContext { } @Override - public List<ILSMComponent> getComponentsToBeMerged() { + public List<ILSMDiskComponent> getComponentsToBeMerged() { return componentsToBeMerged; } @@ -129,7 +130,7 @@ public class ExternalRTreeOpContext extends AbstractLSMIndexOperationContext { } @Override - public List<ILSMComponent> getComponentsToBeReplicated() { + public List<ILSMDiskComponent> getComponentsToBeReplicated() { return componentsToBeReplicated; } }
