http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java index eb3924c..5a773cc 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java @@ -19,9 +19,7 @@ package org.apache.hyracks.storage.am.lsm.invertedindex.impls; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -29,13 +27,9 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.data.std.primitive.IntegerPointable; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification; import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType; import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor; @@ -61,11 +55,9 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy; import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker; import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; -import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType; import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext; -import org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager; import org.apache.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor; @@ -73,13 +65,11 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInvertedIndexAccessor; import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex; -import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedIndexSearchPredicate; import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexUtils; import org.apache.hyracks.storage.common.ICursorInitialState; import org.apache.hyracks.storage.common.IIndexAccessor; -import org.apache.hyracks.storage.common.IIndexBulkLoader; import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.ISearchOperationCallback; @@ -93,10 +83,6 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex protected final IBinaryTokenizerFactory tokenizerFactory; - // On-disk components. - // For creating inverted indexes in flush and merge. - protected final ILSMDiskComponentFactory componentFactory; - // Type traits and comparators for tokens and inverted-list elements. protected final ITypeTraits[] invListTypeTraits; protected final IBinaryComparatorFactory[] invListCmpFactories; @@ -106,18 +92,17 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex private final int[] invertedIndexFieldsForNonBulkLoadOps; public LSMInvertedIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, - OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory, - BloomFilterFactory bloomFilterFactory, IComponentFilterHelper filterHelper, + ILSMDiskComponentFactory componentFactory, IComponentFilterHelper filterHelper, ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, - double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager, ITypeTraits[] invListTypeTraits, - IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, - IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, - ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, - ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, int[] filterFields, - int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) - throws HyracksDataException { - super(ioManager, virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager, - bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory, + double bloomFilterFalsePositiveRate, IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, + ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, + ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, + IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, + ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, + int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, + boolean durable) throws HyracksDataException { + super(ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy, + opTracker, ioScheduler, ioOpCallback, componentFactory, componentFactory, filterFrameFactory, filterManager, filterFields, durable, filterHelper, invertedIndexFields, ITracer.NONE); this.tokenizerFactory = tokenizerFactory; this.invListTypeTraits = invListTypeTraits; @@ -126,8 +111,6 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex this.tokenCmpFactories = tokenCmpFactories; this.filterFieldsForNonBulkLoadOps = filterFieldsForNonBulkLoadOps; this.invertedIndexFieldsForNonBulkLoadOps = invertedIndexFieldsForNonBulkLoadOps; - componentFactory = new LSMInvertedIndexDiskComponentFactory(diskInvIndexFactory, deletedKeysBTreeFactory, - bloomFilterFactory, filterHelper); int i = 0; for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) { @@ -145,52 +128,6 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex } } - @Override - protected ILSMDiskComponent loadComponent(LSMComponentFileReferences refs) throws HyracksDataException { - return createDiskInvIndexComponent(componentFactory, refs.getInsertIndexFileReference(), - refs.getDeleteIndexFileReference(), refs.getBloomFilterFileReference(), false); - } - - @Override - protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException { - LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c; - component.getBloomFilter().deactivate(); - component.getInvIndex().deactivate(); - component.getDeletedKeysBTree().deactivate(); - component.getBloomFilter().destroy(); - component.getInvIndex().destroy(); - component.getDeletedKeysBTree().destroy(); - } - - @Override - protected void deactivateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { - LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c; - mutableComponent.getInvIndex().deactivate(); - mutableComponent.getDeletedKeysBTree().deactivate(); - mutableComponent.getInvIndex().destroy(); - mutableComponent.getDeletedKeysBTree().destroy(); - ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).close(); - } - - @Override - protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException { - LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c; - component.getBloomFilter().deactivate(); - component.getBloomFilter().purge(); - component.getInvIndex().deactivate(); - component.getInvIndex().purge(); - component.getDeletedKeysBTree().deactivate(); - component.getDeletedKeysBTree().purge(); - } - - @Override - protected void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException { - LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c; - component.getInvIndex().destroy(); - component.getDeletedKeysBTree().destroy(); - component.getBloomFilter().destroy(); - } - /** * The keys in the in-memory deleted-keys BTree only refer to on-disk components. * We delete documents from the in-memory inverted index by deleting its entries directly, @@ -265,17 +202,17 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex ILSMComponent component = operationalComponents.get(i); if (component.getType() == LSMComponentType.MEMORY) { includeMutableComponent = true; - IIndexAccessor invIndexAccessor = ((LSMInvertedIndexMemoryComponent) component).getInvIndex() + IIndexAccessor invIndexAccessor = component.getIndex() .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); indexAccessors.add(invIndexAccessor); - IIndexAccessor deletedKeysAccessor = ((LSMInvertedIndexMemoryComponent) component).getDeletedKeysBTree() + IIndexAccessor deletedKeysAccessor = ((LSMInvertedIndexMemoryComponent) component).getBuddyIndex() .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); deletedKeysBTreeAccessors.add(deletedKeysAccessor); } else { - IIndexAccessor invIndexAccessor = ((LSMInvertedIndexDiskComponent) component).getInvIndex() + IIndexAccessor invIndexAccessor = component.getIndex() .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); indexAccessors.add(invIndexAccessor); - IIndexAccessor deletedKeysAccessor = ((LSMInvertedIndexDiskComponent) component).getDeletedKeysBTree() + IIndexAccessor deletedKeysAccessor = ((LSMInvertedIndexDiskComponent) component).getBuddyIndex() .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); deletedKeysBTreeAccessors.add(deletedKeysAccessor); } @@ -299,16 +236,16 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex initState = new LSMInvertedIndexSearchCursorInitialState(keyCmp, keysOnlyTuple, indexAccessors, deletedKeysBTreeAccessors, ((LSMInvertedIndexMemoryComponent) memoryComponents.get(currentMutableComponentId.get())) - .getDeletedKeysBTree().getLeafFrameFactory(), + .getBuddyIndex().getLeafFrameFactory(), ictx, includeMutableComponent, getLsmHarness(), operationalComponents); } else { LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) memoryComponents.get(currentMutableComponentId.get()); - InMemoryInvertedIndex memInvIndex = (InMemoryInvertedIndex) mutableComponent.getInvIndex(); - MultiComparator tokensAndKeysCmp = MultiComparator.create(memInvIndex.getBTree().getComparatorFactories()); + MultiComparator tokensAndKeysCmp = + MultiComparator.create(mutableComponent.getIndex().getBTree().getComparatorFactories()); initState = new LSMInvertedIndexRangeSearchCursorInitialState(tokensAndKeysCmp, keyCmp, keysOnlyTuple, ((LSMInvertedIndexMemoryComponent) memoryComponents.get(currentMutableComponentId.get())) - .getDeletedKeysBTree().getLeafFrameFactory(), + .getBuddyIndex().getLeafFrameFactory(), includeMutableComponent, getLsmHarness(), indexAccessors, deletedKeysBTreeAccessors, pred, operationalComponents); } @@ -334,7 +271,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex LSMInvertedIndexFlushOperation flushOp = (LSMInvertedIndexFlushOperation) operation; // Create an inverted index instance to be bulk loaded. - LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory, flushOp.getTarget(), + ILSMDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getDeletedKeysBTreeTarget(), flushOp.getBloomFilterTarget(), true); // Create a scan cursor on the BTree underlying the in-memory inverted index. @@ -344,9 +281,9 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null); // Search the deleted keys BTree to calculate the number of elements for BloomFilter - IIndexAccessor deletedKeysBTreeAccessor = flushingComponent.getDeletedKeysBTree() + BTreeAccessor deletedKeysBTreeAccessor = flushingComponent.getBuddyIndex() .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); - IIndexCursor btreeCountingCursor = ((BTreeAccessor) deletedKeysBTreeAccessor).createCountingSearchCursor(); + IIndexCursor btreeCountingCursor = deletedKeysBTreeAccessor.createCountingSearchCursor(); deletedKeysBTreeAccessor.search(btreeCountingCursor, nullPred); long numBTreeTuples = 0L; try { @@ -360,7 +297,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex } ILSMDiskComponentBulkLoader componentBulkLoader = - createComponentBulkLoader(component, 1.0f, false, numBTreeTuples, false, false, false); + component.createBulkLoader(1.0f, false, numBTreeTuples, false, false, false); // Create a scan cursor on the deleted keys BTree underlying the in-memory inverted index. IIndexCursor deletedKeysScanCursor = deletedKeysBTreeAccessor.createSearchCursor(false); @@ -369,16 +306,15 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex try { while (deletedKeysScanCursor.hasNext()) { deletedKeysScanCursor.next(); - ((LSMInvertedIndexDiskComponentBulkLoader) componentBulkLoader) - .delete(deletedKeysScanCursor.getTuple()); + componentBulkLoader.delete(deletedKeysScanCursor.getTuple()); } } finally { deletedKeysScanCursor.close(); } // Scan the in-memory inverted index - InMemoryInvertedIndexAccessor memInvIndexAccessor = (InMemoryInvertedIndexAccessor) flushingComponent - .getInvIndex().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); + InMemoryInvertedIndexAccessor memInvIndexAccessor = flushingComponent.getIndex() + .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); BTreeAccessor memBTreeAccessor = memInvIndexAccessor.getBTreeAccessor(); IIndexCursor scanCursor = memBTreeAccessor.createSearchCursor(false); memBTreeAccessor.search(scanCursor, nullPred); @@ -397,8 +333,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple()); filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple()); filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples); - filterManager.writeFilter(component.getLSMComponentFilter(), - ((OnDiskInvertedIndex) component.getInvIndex()).getBTree()); + filterManager.writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder()); } flushingComponent.getMetadata().copy(component.getMetadata()); @@ -418,7 +353,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex search(opCtx, cursor, mergePred); // Create an inverted index instance. - LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory, mergeOp.getTarget(), + ILSMDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getDeletedKeysBTreeTarget(), mergeOp.getBloomFilterTarget(), true); ILSMDiskComponentBulkLoader componentBulkLoader; @@ -439,7 +374,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex .getNumElements(); } - componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numElements, false, false, false); + componentBulkLoader = component.createBulkLoader(1.0f, false, numElements, false, false, false); try { while (btreeCursor.hasNext()) { btreeCursor.next(); @@ -450,7 +385,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex btreeCursor.close(); } } else { - componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, 0L, false, false, false); + componentBulkLoader = component.createBulkLoader(1.0f, false, 0L, false, false, false); } try { @@ -475,8 +410,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex } } getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples); - getFilterManager().writeFilter(component.getLSMComponentFilter(), - ((OnDiskInvertedIndex) component.getInvIndex()).getBTree()); + getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder()); } componentBulkLoader.end(); @@ -484,75 +418,6 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex return component; } - @Override - public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, - boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter, - boolean cleanupEmptyComponent) throws HyracksDataException { - BloomFilterSpecification bloomFilterSpec = null; - if (numElementsHint > 0) { - int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint); - bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate); - } - if (withFilter && filterFields != null) { - return new LSMInvertedIndexDiskComponentBulkLoader((LSMInvertedIndexDiskComponent) component, - bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, cleanupEmptyComponent, - filterManager, treeFields, filterFields, - MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories())); - } else { - return new LSMInvertedIndexDiskComponentBulkLoader((LSMInvertedIndexDiskComponent) component, - bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, - cleanupEmptyComponent); - } - } - - @Override - public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint) - throws HyracksDataException { - return new LSMInvertedIndexBulkLoader(fillFactor, verifyInput, numElementsHint); - } - - public class LSMInvertedIndexBulkLoader implements IIndexBulkLoader { - private final ILSMDiskComponent component; - private final IIndexBulkLoader componentBulkLoader; - - public LSMInvertedIndexBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint) - throws HyracksDataException { - // Note that by using a flush target file name, we state that the - // new bulk loaded tree is "newer" than any other merged tree. - component = createBulkLoadTarget(); - - componentBulkLoader = - createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true, true); - } - - @Override - public void add(ITupleReference tuple) throws HyracksDataException { - componentBulkLoader.add(tuple); - } - - @Override - public void end() throws HyracksDataException { - componentBulkLoader.end(); - if (component.getComponentSize() > 0) { - ioOpCallback.afterOperation(LSMOperationType.FLUSH, null, component); - lsmHarness.addBulkLoadedComponent(component); - } - } - - @Override - public void abort() throws HyracksDataException { - componentBulkLoader.abort(); - } - - } - - @Override - public ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException { - LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference(); - return createDiskInvIndexComponent(componentFactory, componentFileRefs.getInsertIndexFileReference(), - componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true); - } - protected InMemoryInvertedIndex createInMemoryInvertedIndex(IVirtualBufferCache virtualBufferCache, VirtualFreePageManager virtualFreePageManager, int id) throws HyracksDataException { return InvertedIndexUtils.createInMemoryBTreeInvertedindex(virtualBufferCache, virtualFreePageManager, @@ -560,27 +425,6 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_vocab_" + id)); } - protected LSMInvertedIndexDiskComponent createDiskInvIndexComponent(ILSMDiskComponentFactory factory, - FileReference dictBTreeFileRef, FileReference btreeFileRef, FileReference bloomFilterFileRef, - boolean create) throws HyracksDataException { - LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) factory - .createComponent(new LSMComponentFileReferences(dictBTreeFileRef, btreeFileRef, bloomFilterFileRef)); - if (create) { - component.getInvIndex().create(); - component.getDeletedKeysBTree().create(); - component.getBloomFilter().create(); - } - component.getInvIndex().activate(); - component.getDeletedKeysBTree().activate(); - component.getBloomFilter().activate(); - // Will be closed during cleanup of merge(). - if (component.getLSMComponentFilter() != null && !create) { - getFilterManager().readFilter(component.getLSMComponentFilter(), - ((OnDiskInvertedIndex) component.getInvIndex()).getBTree()); - } - return component; - } - @Override public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) throws HyracksDataException { @@ -624,65 +468,12 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex } @Override - public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) { - Set<String> files = new HashSet<>(); - LSMInvertedIndexDiskComponent invIndexComponent = (LSMInvertedIndexDiskComponent) lsmComponent; - OnDiskInvertedIndex invIndex = (OnDiskInvertedIndex) invIndexComponent.getInvIndex(); - files.add(invIndex.getInvListsFile().getFile().getAbsolutePath()); - files.add(invIndex.getBTree().getFileReference().getFile().getAbsolutePath()); - files.add(invIndexComponent.getBloomFilter().getFileReference().getFile().getAbsolutePath()); - files.add(invIndexComponent.getDeletedKeysBTree().getFileReference().getFile().getAbsolutePath()); - return files; - } - - @Override - protected void clearMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { - LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c; - mutableComponent.getInvIndex().clear(); - mutableComponent.getDeletedKeysBTree().clear(); - mutableComponent.reset(); - } - - @Override - protected long getMemoryComponentSize(ILSMMemoryComponent c) { - LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c; - IBufferCache virtualBufferCache = mutableComponent.getInvIndex().getBufferCache(); - return ((long) virtualBufferCache.getNumPages()) * virtualBufferCache.getPageSize(); - } - - @Override - protected void validateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { - LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c; - mutableComponent.getInvIndex().validate(); - mutableComponent.getDeletedKeysBTree().validate(); - } - - @Override - protected void validateDiskComponent(ILSMDiskComponent c) throws HyracksDataException { - LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c; - component.getInvIndex().validate(); - component.getDeletedKeysBTree().validate(); - } - - @Override - protected void allocateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { - LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c; - ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).open(); - mutableComponent.getInvIndex().create(); - mutableComponent.getInvIndex().activate(); - mutableComponent.getDeletedKeysBTree().create(); - mutableComponent.getDeletedKeysBTree().activate(); - } - - @Override protected LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent, ILSMDiskComponent lastComponent) throws HyracksDataException { LSMInvertedIndexDiskComponent first = (LSMInvertedIndexDiskComponent) firstComponent; - OnDiskInvertedIndex firstInvIndex = (OnDiskInvertedIndex) first.getInvIndex(); - String firstFileName = firstInvIndex.getBTree().getFileReference().getFile().getName(); + String firstFileName = first.getMetadataHolder().getFileReference().getFile().getName(); LSMInvertedIndexDiskComponent last = (LSMInvertedIndexDiskComponent) lastComponent; - OnDiskInvertedIndex lastInvIndex = (OnDiskInvertedIndex) last.getInvIndex(); - String lastFileName = lastInvIndex.getBTree().getFileReference().getFile().getName(); + String lastFileName = last.getMetadataHolder().getFileReference().getFile().getName(); return fileManager.getRelMergeFileReference(firstFileName, lastFileName); }
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java index b77f894..279a518 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java @@ -18,57 +18,62 @@ */ package org.apache.hyracks.storage.am.lsm.invertedindex.impls; +import java.util.HashSet; +import java.util.Set; + import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter; import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.common.api.IMetadataPageManager; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter; -import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBuddyDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex; import org.apache.hyracks.storage.am.lsm.common.util.ComponentUtils; -import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -public class LSMInvertedIndexDiskComponent extends AbstractLSMDiskComponent { +public class LSMInvertedIndexDiskComponent extends AbstractLSMWithBuddyDiskComponent { - private final IInPlaceInvertedIndex invIndex; + private final OnDiskInvertedIndex invIndex; private final BTree deletedKeysBTree; private final BloomFilter bloomFilter; - public LSMInvertedIndexDiskComponent(IInPlaceInvertedIndex invIndex, BTree deletedKeysBTree, - BloomFilter bloomFilter, ILSMComponentFilter filter) throws HyracksDataException { - super((IMetadataPageManager) deletedKeysBTree.getPageManager(), filter); + public LSMInvertedIndexDiskComponent(AbstractLSMIndex lsmIndex, OnDiskInvertedIndex invIndex, + BTree deletedKeysBTree, BloomFilter bloomFilter, ILSMComponentFilter filter) { + super(lsmIndex, (IMetadataPageManager) deletedKeysBTree.getPageManager(), filter); this.invIndex = invIndex; this.deletedKeysBTree = deletedKeysBTree; this.bloomFilter = bloomFilter; } @Override - public void destroy() throws HyracksDataException { - invIndex.deactivate(); - invIndex.destroy(); - deletedKeysBTree.deactivate(); - deletedKeysBTree.destroy(); - bloomFilter.deactivate(); - bloomFilter.destroy(); + public OnDiskInvertedIndex getIndex() { + return invIndex; } - public IInPlaceInvertedIndex getInvIndex() { - return invIndex; + @Override + public BTree getMetadataHolder() { + return invIndex.getBTree(); } - public BTree getDeletedKeysBTree() { + @Override + public BTree getBuddyIndex() { return deletedKeysBTree; } + @Override public BloomFilter getBloomFilter() { return bloomFilter; } @Override + public IBufferCache getBloomFilterBufferCache() { + return invIndex.getBufferCache(); + } + + @Override public long getComponentSize() { - return ((OnDiskInvertedIndex) invIndex).getInvListsFile().getFile().length() - + ((OnDiskInvertedIndex) invIndex).getBTree().getFileReference().getFile().length() + return invIndex.getInvListsFile().getFile().length() + invIndex.getBTree().getFileReference().getFile().length() + deletedKeysBTree.getFileReference().getFile().length() + bloomFilter.getFileReference().getFile().length(); } @@ -79,20 +84,29 @@ public class LSMInvertedIndexDiskComponent extends AbstractLSMDiskComponent { } @Override + public Set<String> getLSMComponentPhysicalFiles() { + Set<String> files = new HashSet<>(); + files.add(invIndex.getInvListsFile().getFile().getAbsolutePath()); + files.add(invIndex.getBTree().getFileReference().getFile().getAbsolutePath()); + files.add(bloomFilter.getFileReference().getFile().getAbsolutePath()); + files.add(deletedKeysBTree.getFileReference().getFile().getAbsolutePath()); + return files; + } + + @Override public String toString() { - return getClass().getSimpleName() + ":" + ((OnDiskInvertedIndex) invIndex).getInvListsFile().getRelativePath(); + return getClass().getSimpleName() + ":" + invIndex.getInvListsFile().getRelativePath(); } @Override public void markAsValid(boolean persist) throws HyracksDataException { - IBufferCache bufferCache = invIndex.getBufferCache(); - ComponentUtils.markAsValid(invIndex.getBufferCache(), bloomFilter, persist); + ComponentUtils.markAsValid(getBloomFilterBufferCache(), getBloomFilter(), persist); // Flush inverted index second. - bufferCache.force(((OnDiskInvertedIndex) invIndex).getInvListsFileId(), true); - ComponentUtils.markAsValid(((OnDiskInvertedIndex) invIndex).getBTree(), persist); + invIndex.getBufferCache().force((invIndex).getInvListsFileId(), true); + ComponentUtils.markAsValid(getMetadataHolder(), persist); // Flush deleted keys BTree. - ComponentUtils.markAsValid(deletedKeysBTree, persist); + ComponentUtils.markAsValid(getBuddyIndex(), persist); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java deleted file mode 100644 index daa93ae..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hyracks.storage.am.lsm.invertedindex.impls; - -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification; -import org.apache.hyracks.storage.am.common.api.ITreeIndex; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterManager; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; -import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponentWithBuddyBulkLoader; -import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex; -import org.apache.hyracks.storage.common.IIndex; -import org.apache.hyracks.storage.common.MultiComparator; - -public class LSMInvertedIndexDiskComponentBulkLoader extends AbstractLSMDiskComponentWithBuddyBulkLoader { - - //with filter - public LSMInvertedIndexDiskComponentBulkLoader(LSMInvertedIndexDiskComponent component, - BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint, - boolean checkIfEmptyIndex, boolean cleanupEmptyComponent, ILSMComponentFilterManager filterManager, - int[] indexFields, int[] filterFields, MultiComparator filterCmp) throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, - cleanupEmptyComponent, filterManager, indexFields, filterFields, filterCmp); - } - - //without filter - public LSMInvertedIndexDiskComponentBulkLoader(LSMInvertedIndexDiskComponent component, - BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint, - boolean checkIfEmptyIndex, boolean cleanupEmptyComponent) throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, - cleanupEmptyComponent, null, null, null, null); - } - - @Override - protected BloomFilter getBloomFilter(ILSMDiskComponent component) { - return ((LSMInvertedIndexDiskComponent) component).getBloomFilter(); - } - - @Override - protected IIndex getIndex(ILSMDiskComponent component) { - return ((LSMInvertedIndexDiskComponent) component).getInvIndex(); - } - - @Override - protected ITreeIndex getTreeIndex(ILSMDiskComponent component) { - return ((OnDiskInvertedIndex) ((LSMInvertedIndexDiskComponent) component).getInvIndex()).getBTree(); - } - - @Override - protected ITreeIndex getBuddyBTree(ILSMDiskComponent component) { - return ((LSMInvertedIndexDiskComponent) component).getDeletedKeysBTree(); - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java index d018535..253ed20 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java @@ -24,6 +24,7 @@ import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory; import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexFactory; @@ -44,8 +45,9 @@ public class LSMInvertedIndexDiskComponentFactory implements ILSMDiskComponentFa } @Override - public LSMInvertedIndexDiskComponent createComponent(LSMComponentFileReferences cfr) throws HyracksDataException { - return new LSMInvertedIndexDiskComponent( + public LSMInvertedIndexDiskComponent createComponent(AbstractLSMIndex lsmIndex, LSMComponentFileReferences cfr) + throws HyracksDataException { + return new LSMInvertedIndexDiskComponent(lsmIndex, diskInvIndexFactory.createIndexInstance(cfr.getInsertIndexFileReference()), btreeFactory.createIndexInstance(cfr.getDeleteIndexFileReference()), bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()), http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java index c164cf9..2d2111c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java @@ -19,43 +19,31 @@ package org.apache.hyracks.storage.am.lsm.invertedindex.impls; -import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter; +import org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBuddyMemoryComponent; import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; -import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMMemoryComponent; -import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex; +import org.apache.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInvertedIndex; -public class LSMInvertedIndexMemoryComponent extends AbstractLSMMemoryComponent { +public class LSMInvertedIndexMemoryComponent extends AbstractLSMWithBuddyMemoryComponent { - private final IInvertedIndex invIndex; + private final InMemoryInvertedIndex invIndex; private final BTree deletedKeysBTree; - public LSMInvertedIndexMemoryComponent(IInvertedIndex invIndex, BTree deletedKeysBTree, IVirtualBufferCache vbc, - boolean isActive, ILSMComponentFilter filter) { + public LSMInvertedIndexMemoryComponent(InMemoryInvertedIndex invIndex, BTree deletedKeysBTree, + IVirtualBufferCache vbc, boolean isActive, ILSMComponentFilter filter) { super(vbc, isActive, filter); this.invIndex = invIndex; this.deletedKeysBTree = deletedKeysBTree; } - public IInvertedIndex getInvIndex() { + @Override + public InMemoryInvertedIndex getIndex() { return invIndex; } - public BTree getDeletedKeysBTree() { - return deletedKeysBTree; - } - @Override - public void reset() throws HyracksDataException { - super.reset(); - invIndex.deactivate(); - invIndex.destroy(); - invIndex.create(); - invIndex.activate(); - deletedKeysBTree.deactivate(); - deletedKeysBTree.destroy(); - deletedKeysBTree.create(); - deletedKeysBTree.activate(); + public BTree getBuddyIndex() { + return deletedKeysBTree; } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java index c725501..30300d3 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java @@ -28,8 +28,6 @@ import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor; -import org.apache.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInvertedIndex; -import org.apache.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInvertedIndexAccessor; import org.apache.hyracks.storage.common.IIndexAccessor; import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.ISearchOperationCallback; @@ -58,19 +56,17 @@ public class LSMInvertedIndexOpContext extends AbstractLSMIndexOperationContext LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) mutableComponents.get(i); if (allFields != null) { - mutableInvIndexAccessors[i] = (IInvertedIndexAccessor) ((InMemoryInvertedIndex) mutableComponent - .getInvIndex()).createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE, - allFields); + mutableInvIndexAccessors[i] = mutableComponent.getIndex().createAccessor(allFields); } else { - mutableInvIndexAccessors[i] = (IInvertedIndexAccessor) mutableComponent.getInvIndex() - .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); + mutableInvIndexAccessors[i] = mutableComponent.getIndex().createAccessor(NoOpOperationCallback.INSTANCE, + NoOpOperationCallback.INSTANCE); } - deletedKeysBTreeAccessors[i] = mutableComponent.getDeletedKeysBTree() + deletedKeysBTreeAccessors[i] = mutableComponent.getBuddyIndex() .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); } // Project away the document fields, leaving only the key fields. LSMInvertedIndexMemoryComponent c = (LSMInvertedIndexMemoryComponent) mutableComponents.get(0); - int numKeyFields = c.getInvIndex().getInvListTypeTraits().length; + int numKeyFields = c.getIndex().getInvListTypeTraits().length; int[] keyFieldPermutation = new int[numKeyFields]; for (int i = 0; i < numKeyFields; i++) { keyFieldPermutation[i] = NUM_DOCUMENT_FIELDS + i; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java index d154356..b3a63af 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java @@ -25,9 +25,9 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.IIOManager; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory; import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager; @@ -35,31 +35,29 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy; import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker; import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager; -import org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager; import org.apache.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInvertedIndex; -import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexUtils; +import org.apache.hyracks.storage.common.buffercache.IBufferCache; public class PartitionedLSMInvertedIndex extends LSMInvertedIndex { public PartitionedLSMInvertedIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, - OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory, - BloomFilterFactory bloomFilterFactory, IComponentFilterHelper filterHelper, + ILSMDiskComponentFactory componentFactory, IComponentFilterHelper filterHelper, ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, - double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager, ITypeTraits[] invListTypeTraits, - IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, - IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, - ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, - ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, int[] filterFields, - int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) - throws HyracksDataException { - super(ioManager, virtualBufferCaches, diskInvIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, - filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, fileManager, - invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, - mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, - filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable); + double bloomFilterFalsePositiveRate, IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, + ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, + ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, + IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, + ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, + int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, + boolean durable) throws HyracksDataException { + super(ioManager, virtualBufferCaches, componentFactory, filterHelper, filterFrameFactory, filterManager, + bloomFilterFalsePositiveRate, diskBufferCache, fileManager, invListTypeTraits, invListCmpFactories, + tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, + invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, + durable); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java index 54cae72..cf40a7a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java @@ -34,7 +34,6 @@ import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor; import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory; -import org.apache.hyracks.storage.common.IIndexAccessor; import org.apache.hyracks.storage.common.IIndexBulkLoader; import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.ISearchOperationCallback; @@ -167,14 +166,13 @@ public class InMemoryInvertedIndex implements IInPlaceInvertedIndex { } @Override - public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, + public InMemoryInvertedIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) throws HyracksDataException { return new InMemoryInvertedIndexAccessor(this, new InMemoryInvertedIndexOpContext(btree, tokenCmpFactories, tokenizerFactory)); } - public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, - ISearchOperationCallback searchCallback, int[] nonIndexFields) throws HyracksDataException { + public InMemoryInvertedIndexAccessor createAccessor(int[] nonIndexFields) throws HyracksDataException { return new InMemoryInvertedIndexAccessor(this, new InMemoryInvertedIndexOpContext(btree, tokenCmpFactories, tokenizerFactory), nonIndexFields); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java index 2e0511c..73f90a6 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java @@ -51,8 +51,8 @@ public class InMemoryInvertedIndexAccessor implements IInvertedIndexAccessor { this.opCtx = opCtx; this.index = index; this.searcher = createSearcher(); - this.btreeAccessor = (BTreeAccessor) index.getBTree().createAccessor(NoOpOperationCallback.INSTANCE, - NoOpOperationCallback.INSTANCE); + this.btreeAccessor = + index.getBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); } public InMemoryInvertedIndexAccessor(InMemoryInvertedIndex index, IIndexOperationContext opCtx, @@ -60,7 +60,7 @@ public class InMemoryInvertedIndexAccessor implements IInvertedIndexAccessor { this.opCtx = opCtx; this.index = index; this.searcher = createSearcher(); - this.btreeAccessor = (BTreeAccessor) index.getBTree().createAccessor(NoOpOperationCallback.INSTANCE, + this.btreeAccessor = index.getBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE, nonIndexFields); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexOpContext.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexOpContext.java index c37f880..90dbc81 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexOpContext.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexOpContext.java @@ -67,8 +67,8 @@ public class InMemoryInvertedIndexOpContext implements IIndexOperationContext { case SEARCH: { if (getBtreePred() == null) { btreePred = new RangePredicate(null, null, true, true, null, null); - btreeAccessor = (BTreeAccessor) btree.createAccessor(NoOpOperationCallback.INSTANCE, - NoOpOperationCallback.INSTANCE); + btreeAccessor = + btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); btreeCmp = MultiComparator.create(btree.getComparatorFactories()); tokenFieldsCmp = MultiComparator.create(tokenCmpFactories); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java index e615bf0..933ef21 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java @@ -37,7 +37,6 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedListPartit import org.apache.hyracks.storage.am.lsm.invertedindex.search.PartitionedTOccurrenceSearcher; import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.util.PartitionedInvertedIndexTokenizingTupleIterator; -import org.apache.hyracks.storage.common.IIndexAccessor; import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.buffercache.IBufferCache; @@ -88,15 +87,14 @@ public class PartitionedInMemoryInvertedIndex extends InMemoryInvertedIndex impl } @Override - public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, + public PartitionedInMemoryInvertedIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) throws HyracksDataException { return new PartitionedInMemoryInvertedIndexAccessor(this, new PartitionedInMemoryInvertedIndexOpContext(btree, tokenCmpFactories, tokenizerFactory)); } @Override - public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, - ISearchOperationCallback searchCallback, int[] nonIndexFields) throws HyracksDataException { + public PartitionedInMemoryInvertedIndexAccessor createAccessor(int[] nonIndexFields) throws HyracksDataException { return new PartitionedInMemoryInvertedIndexAccessor(this, new PartitionedInMemoryInvertedIndexOpContext(btree, tokenCmpFactories, tokenizerFactory), nonIndexFields); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java index 5d8cafb..6244ee7 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java @@ -486,7 +486,7 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex { } @Override - public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, + public OnDiskInvertedIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) throws HyracksDataException { return new OnDiskInvertedIndexAccessor(this); } @@ -554,7 +554,7 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex { PermutingTupleReference tokenTuple = new PermutingTupleReference(fieldPermutation); IInvertedIndexAccessor invIndexAccessor = - (IInvertedIndexAccessor) createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); + createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); IInvertedListCursor invListCursor = invIndexAccessor.createInvertedListCursor(); MultiComparator invListCmp = MultiComparator.create(invListCmpFactories); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java index 0edc63c..f058e4a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java @@ -26,7 +26,6 @@ import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; import org.apache.hyracks.storage.am.lsm.common.impls.IndexFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex; -import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNameMapper; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilderFactory; @@ -56,7 +55,7 @@ public class OnDiskInvertedIndexFactory extends IndexFactory<IInvertedIndex> { } @Override - public IInPlaceInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws HyracksDataException { + public OnDiskInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws HyracksDataException { String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getAbsolutePath()); FileReference invListsFile = ioManager.resolveAbsolutePath(invListsFilePath); IInvertedListBuilder invListBuilder = invListBuilderFactory.create(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java index 0c0110e..569c51d 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java @@ -35,7 +35,6 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IPartitionedInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedListPartitions; import org.apache.hyracks.storage.am.lsm.invertedindex.search.PartitionedTOccurrenceSearcher; -import org.apache.hyracks.storage.common.IIndexAccessor; import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.buffercache.IBufferCache; @@ -59,7 +58,7 @@ public class PartitionedOnDiskInvertedIndex extends OnDiskInvertedIndex implemen } @Override - public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, + public PartitionedOnDiskInvertedIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) throws HyracksDataException { return new PartitionedOnDiskInvertedIndexAccessor(this); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java index e8d12ad..7f10658 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java @@ -24,7 +24,6 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; -import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNameMapper; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilderFactory; @@ -42,7 +41,7 @@ public class PartitionedOnDiskInvertedIndexFactory extends OnDiskInvertedIndexFa } @Override - public IInPlaceInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws HyracksDataException { + public PartitionedOnDiskInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws HyracksDataException { String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getAbsolutePath()); FileReference invListsFile = ioManager.resolveAbsolutePath(invListsFilePath); IInvertedListBuilder invListBuilder = invListBuilderFactory.create(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java index 48afdbc3..8f9b5de 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java @@ -36,6 +36,7 @@ import org.apache.hyracks.storage.am.common.api.IPageManager; import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy; @@ -48,6 +49,7 @@ import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilderFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndex; +import org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexFileManager; import org.apache.hyracks.storage.am.lsm.invertedindex.impls.PartitionedLSMInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInvertedIndex; @@ -157,11 +159,14 @@ public class InvertedIndexUtils { filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory); filterManager = new LSMComponentFilterManager(filterFrameFactory); } - return new LSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory, - bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, - fileManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, - tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, - filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable); + ILSMDiskComponentFactory componentFactory = new LSMInvertedIndexDiskComponentFactory(invIndexFactory, + deletedKeysBTreeFactory, bloomFilterFactory, filterHelper); + + return new LSMInvertedIndex(ioManager, virtualBufferCaches, componentFactory, filterHelper, filterFrameFactory, + filterManager, bloomFilterFalsePositiveRate, diskBufferCache, fileManager, invListTypeTraits, + invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, + ioScheduler, ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, + invertedIndexFieldsForNonBulkLoadOps, durable); } public static PartitionedLSMInvertedIndex createPartitionedLSMInvertedIndex(IIOManager ioManager, @@ -203,10 +208,13 @@ public class InvertedIndexUtils { filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory); filterManager = new LSMComponentFilterManager(filterFrameFactory); } - return new PartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory, - bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, - fileManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, - tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, + ILSMDiskComponentFactory componentFactory = new LSMInvertedIndexDiskComponentFactory(invIndexFactory, + deletedKeysBTreeFactory, bloomFilterFactory, filterHelper); + + return new PartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, componentFactory, filterHelper, + filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, diskBufferCache, fileManager, + invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, + mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable); } }
