Luo Chen has submitted this change and it was merged. Change subject: [NO ISSUE][STO] Misc fixes of inverted index ......................................................................
[NO ISSUE][STO] Misc fixes of inverted index - user model changes: no - storage format changes: no - interface changes: no Details: This patch mainly improves the inverted index merge perf in the following ways: - Fix the bloom filter check on deleted btree - Remove unnecessary btree search during inverted index range search - Remove unnecessary page pins when merging inverted lists by making the cursor remembering the last pinned page - Fix close/destroy operations of range search cursor Change-Id: I7bbf6fe2d332db0e0368a4ce0f1ab218ba6da81d Reviewed-on: https://asterix-gerrit.ics.uci.edu/2514 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Contrib: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: abdullah alamoudi <[email protected]> --- M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java 13 files changed, 154 insertions(+), 124 deletions(-) Approvals: Anon. E. Moose #1000171: abdullah alamoudi: Looks good to me, approved Jenkins: Verified; No violations found; ; Verified diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java index 18de540..c635411 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java @@ -19,8 +19,6 @@ package org.apache.hyracks.storage.am.common.tuples; -import java.util.Arrays; - import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; /** @@ -96,14 +94,17 @@ return tuples[tupleIndex].getFieldLength(fieldIndex); } + /** + * Right now this class is only used by inverted index, and only contains 2 tuples. + * As a result, sequential search would be more efficient than binary search + */ private int getTupleIndex(int fIdx) { - int tupleIndex = Arrays.binarySearch(fieldCounts, 0, numTuples - 1, fIdx); - if (tupleIndex < 0) { - tupleIndex = -tupleIndex - 1; - } else { - ++tupleIndex; + for (int i = 0; i < numTuples; i++) { + if (fIdx < fieldCounts[i]) { + return i; + } } - return tupleIndex; + throw new IllegalArgumentException("Illegal field index " + fIdx); } private int getFieldIndex(int tupleIndex, int fIdx) { diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java index 445a005..fb1fa63 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java @@ -125,7 +125,6 @@ rangeCursors[i].close(); } } - rangeCursors = null; } finally { if (lsmHarness != null) { lsmHarness.endSearch(opCtx); @@ -249,7 +248,7 @@ public class PriorityQueueElement { private ITupleReference tuple; - private int cursorIndex; + private final int cursorIndex; public PriorityQueueElement(int cursorIndex) { tuple = null; diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java index 4eb7728..f2fb9af 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java @@ -95,14 +95,14 @@ opTrackerProvider.getOperationTracker(serviceCtx, this), ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable, - metadataPageManagerFactory); + metadataPageManagerFactory, serviceCtx.getTracer()); } else { return InvertedIndexUtils.createLSMInvertedIndex(ioManager, virtualBufferCaches, typeTraits, cmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache, file.getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy, opTrackerProvider.getOperationTracker(serviceCtx, this), ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable, - metadataPageManagerFactory); + metadataPageManagerFactory, serviceCtx.getTracer()); } } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java index 5fda514..e46c24a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java @@ -100,10 +100,10 @@ IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] invertedIndexFields, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, - int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) throws HyracksDataException { + int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, ITracer tracer) throws HyracksDataException { super(ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory, componentFactory, componentFactory, filterFrameFactory, - filterManager, filterFields, durable, filterHelper, invertedIndexFields, ITracer.NONE); + filterManager, filterFields, durable, filterHelper, invertedIndexFields, tracer); this.tokenizerFactory = tokenizerFactory; this.invListTypeTraits = invListTypeTraits; this.invListCmpFactories = invListCmpFactories; @@ -349,65 +349,65 @@ LSMInvertedIndexMergeOperation mergeOp = (LSMInvertedIndexMergeOperation) operation; RangePredicate mergePred = new RangePredicate(null, null, true, true, null, null); IIndexCursor cursor = mergeOp.getCursor(); - try { - ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx(); - // Scan diskInvertedIndexes ignoring the memoryInvertedIndex. - // Create an inverted index instance. - ILSMDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(), - mergeOp.getDeletedKeysBTreeTarget(), mergeOp.getBloomFilterTarget(), true); - ILSMDiskComponentBulkLoader componentBulkLoader; - // In case we must keep the deleted-keys BTrees, then they must be merged *before* merging the inverted - // indexes so that lsmHarness.endSearch() is called once when the inverted indexes have been merged. - if (mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents - .get(diskComponents.size() - 1)) { - // Keep the deleted tuples since the oldest disk component is not included in the merge operation - LSMInvertedIndexDeletedKeysBTreeMergeCursor btreeCursor = - new LSMInvertedIndexDeletedKeysBTreeMergeCursor(opCtx); - try { - long numElements = 0L; - for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) { - numElements += ((LSMInvertedIndexDiskComponent) mergeOp.getMergingComponents().get(i)) - .getBloomFilter().getNumElements(); - } - componentBulkLoader = component.createBulkLoader(1.0f, false, numElements, false, false, false); - loadDeleteTuples(opCtx, btreeCursor, mergePred, componentBulkLoader); - } finally { - btreeCursor.destroy(); - } - } else { - componentBulkLoader = component.createBulkLoader(1.0f, false, 0L, false, false, false); - } - search(opCtx, cursor, mergePred); + ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx(); + // Scan diskInvertedIndexes ignoring the memoryInvertedIndex. + // Create an inverted index instance. + ILSMDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(), + mergeOp.getDeletedKeysBTreeTarget(), mergeOp.getBloomFilterTarget(), true); + ILSMDiskComponentBulkLoader componentBulkLoader; + // In case we must keep the deleted-keys BTrees, then they must be merged *before* merging the inverted + // indexes so that lsmHarness.endSearch() is called once when the inverted indexes have been merged. + if (mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents + .get(diskComponents.size() - 1)) { + // Keep the deleted tuples since the oldest disk component is not included in the merge operation + LSMInvertedIndexDeletedKeysBTreeMergeCursor btreeCursor = + new LSMInvertedIndexDeletedKeysBTreeMergeCursor(opCtx); try { - while (cursor.hasNext()) { - cursor.next(); - ITupleReference tuple = cursor.getTuple(); - componentBulkLoader.add(tuple); - } - } finally { - cursor.close(); - } - if (component.getLSMComponentFilter() != null) { - List<ITupleReference> filterTuples = new ArrayList<>(); + long numElements = 0L; for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) { - ITupleReference min = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple(); - ITupleReference max = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple(); - if (min != null) { - filterTuples.add(min); - } - if (max != null) { - filterTuples.add(max); - } + numElements += ((LSMInvertedIndexDiskComponent) mergeOp.getMergingComponents().get(i)) + .getBloomFilter().getNumElements(); } - getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples, - NoOpOperationCallback.INSTANCE); - getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder()); + componentBulkLoader = component.createBulkLoader(1.0f, false, numElements, false, false, false); + loadDeleteTuples(opCtx, btreeCursor, mergePred, componentBulkLoader); + } finally { + btreeCursor.destroy(); } - componentBulkLoader.end(); - return component; - } finally { - cursor.close(); + } else { + componentBulkLoader = component.createBulkLoader(1.0f, false, 0L, false, false, false); } + search(opCtx, cursor, mergePred); + try { + while (cursor.hasNext()) { + cursor.next(); + componentBulkLoader.add(cursor.getTuple()); + } + } finally { + try { + cursor.close(); + } finally { + cursor.destroy(); + } + } + if (component.getLSMComponentFilter() != null) { + List<ITupleReference> filterTuples = new ArrayList<>(); + for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) { + ITupleReference min = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple(); + ITupleReference max = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple(); + if (min != null) { + filterTuples.add(min); + } + if (max != null) { + filterTuples.add(max); + } + } + getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples, + NoOpOperationCallback.INSTANCE); + getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder()); + } + componentBulkLoader.end(); + + return component; } private void loadDeleteTuples(ILSMIndexOperationContext opCtx, diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java index 12dc23f..020b788 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java @@ -98,10 +98,9 @@ keysOnlyTuple.reset(checkElement.getTuple()); int end = checkElement.getCursorIndex(); for (int i = 0; i < end; i++) { - if (bloomFilters[i] != null && bloomFilters[i].contains(keysOnlyTuple, hashes)) { + if (bloomFilters[i] != null && !bloomFilters[i].contains(keysOnlyTuple, hashes)) { continue; } - deletedKeysBTreeCursors[i].close(); deletedKeysBTreeAccessors.get(i).search(deletedKeysBTreeCursors[i], keySearchPred); try { if (deletedKeysBTreeCursors[i].hasNext()) { @@ -114,4 +113,31 @@ return false; } + @Override + public void doClose() throws HyracksDataException { + try { + super.doClose(); + } finally { + if (deletedKeysBTreeCursors != null) { + for (int i = 0; i < deletedKeysBTreeCursors.length; i++) { + deletedKeysBTreeCursors[i].close(); + } + } + } + } + + @Override + public void doDestroy() throws HyracksDataException { + try { + super.doDestroy(); + } finally { + if (deletedKeysBTreeCursors != null) { + for (int i = 0; i < deletedKeysBTreeCursors.length; i++) { + deletedKeysBTreeCursors[i].destroy(); + } + deletedKeysBTreeCursors = null; + } + } + } + } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java index d7482a8..604d374 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java @@ -40,6 +40,7 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexUtils; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.util.trace.ITracer; public class PartitionedLSMInvertedIndex extends LSMInvertedIndex { @@ -52,12 +53,12 @@ IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] invertedIndexFields, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, - int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) throws HyracksDataException { + int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, ITracer tracer) throws HyracksDataException { super(ioManager, virtualBufferCaches, componentFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, diskBufferCache, fileManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, - invertedIndexFieldsForNonBulkLoadOps, durable); + invertedIndexFieldsForNonBulkLoadOps, durable, tracer); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java index b033b99..2401c67 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java @@ -53,6 +53,7 @@ protected ICachedPage page; protected boolean pinned; + protected int pinnedPageId = -1; public FixedSizeElementInvertedListScanCursor(IBufferCache bufferCache, int fileId, ITypeTraits[] invListFields) throws HyracksDataException { @@ -103,14 +104,18 @@ */ @Override public void loadPages() throws HyracksDataException { - if (pinned) { - unloadPages(); - } if (currentPageId == endPageId) { + // inverted list exhausted, return return; } currentPageId++; + if (pinned && pinnedPageId == currentPageId) { + // already pinned, return + return; + } + unloadPages(); page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false); + pinnedPageId = currentPageId; pinned = true; } @@ -134,7 +139,6 @@ // Deducts 1 since the startPage would be set to bufferCurrentPageId + 1 in loadPages(). this.currentPageId = startPageId - 1; this.numPages = endPageId - startPageId + 1; - this.pinned = false; } @Override @@ -158,16 +162,14 @@ @Override public void doClose() throws HyracksDataException { - if (pinned) { - unloadPages(); - } + // No op + // We allow the inverted list cursor to hold at most one page to avoid + // unnecessary pins } @Override public void doDestroy() throws HyracksDataException { - if (pinned) { - unloadPages(); - } + unloadPages(); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java index 2f4f1d6..14ebe46 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java @@ -204,9 +204,9 @@ try { if (ctx.getBtreeCursor().hasNext()) { ctx.getBtreeCursor().next(); - openInvertedListCursor(ctx.getBtreeCursor().getTuple(), listCursor); + openInvertedListCursor(ctx.getBtreeCursor().getTuple(), listCursor, ctx); } else { - LSMInvertedIndexSearchCursorInitialState initState = new LSMInvertedIndexSearchCursorInitialState(); + LSMInvertedIndexSearchCursorInitialState initState = ctx.getCursorInitialState(); initState.setInvertedListInfo(0, 0, 0, 0); listCursor.open(initState, null); } @@ -215,8 +215,8 @@ } } - public void openInvertedListCursor(ITupleReference btreeTuple, InvertedListCursor listCursor) - throws HyracksDataException { + public void openInvertedListCursor(ITupleReference btreeTuple, InvertedListCursor listCursor, + OnDiskInvertedIndexOpContext opCtx) throws HyracksDataException { int startPageId = IntegerPointable.getInteger(btreeTuple.getFieldData(invListStartPageIdField), btreeTuple.getFieldStart(invListStartPageIdField)); int endPageId = IntegerPointable.getInteger(btreeTuple.getFieldData(invListEndPageIdField), @@ -225,7 +225,7 @@ btreeTuple.getFieldStart(invListStartOffField)); int numElements = IntegerPointable.getInteger(btreeTuple.getFieldData(invListNumElementsField), btreeTuple.getFieldStart(invListNumElementsField)); - LSMInvertedIndexSearchCursorInitialState initState = new LSMInvertedIndexSearchCursorInitialState(); + LSMInvertedIndexSearchCursorInitialState initState = opCtx.getCursorInitialState(); initState.setInvertedListInfo(startPageId, endPageId, startOff, numElements); listCursor.open(initState, null); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java index 267cc79..81bc46b9 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java @@ -25,6 +25,7 @@ import org.apache.hyracks.storage.am.common.api.IIndexOperationContext; import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; +import org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexSearchCursorInitialState; import org.apache.hyracks.storage.common.IIndexAccessor; import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.MultiComparator; @@ -32,12 +33,13 @@ public class OnDiskInvertedIndexOpContext implements IIndexOperationContext { private final RangePredicate btreePred = new RangePredicate(null, null, true, true, null, null); - private IIndexAccessor btreeAccessor; - private IIndexCursor btreeCursor; - private MultiComparator searchCmp; + private final IIndexAccessor btreeAccessor; + private final IIndexCursor btreeCursor; + private final MultiComparator searchCmp; // For prefix search on partitioned indexes. private MultiComparator prefixSearchCmp; private boolean destroyed = false; + private LSMInvertedIndexSearchCursorInitialState cursorInitialState; public OnDiskInvertedIndexOpContext(BTree btree) throws HyracksDataException { // TODO: Ignore opcallbacks for now. @@ -96,4 +98,11 @@ btreeCursor.destroy(); } } + + public LSMInvertedIndexSearchCursorInitialState getCursorInitialState() { + if (cursorInitialState == null) { + cursorInitialState = new LSMInvertedIndexSearchCursorInitialState(); + } + return cursorInitialState; + } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java index d9e7d34..9d99c9e 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java @@ -26,7 +26,6 @@ import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters; import org.apache.hyracks.storage.am.common.tuples.ConcatenatingTupleReference; import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference; -import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor; import org.apache.hyracks.storage.common.EnforcedIndexCursor; import org.apache.hyracks.storage.common.ICursorInitialState; @@ -41,7 +40,7 @@ private final BTree btree; private final IIndexAccessor btreeAccessor; - private final IInPlaceInvertedIndex invIndex; + private final OnDiskInvertedIndex invIndex; private final IIndexOperationContext opCtx; private final InvertedListCursor invListRangeSearchCursor; private boolean isInvListCursorOpen; @@ -50,11 +49,11 @@ private RangePredicate btreePred; private final PermutingTupleReference tokenTuple; - private ConcatenatingTupleReference concatTuple; + private final ConcatenatingTupleReference concatTuple; - public OnDiskInvertedIndexRangeSearchCursor(IInPlaceInvertedIndex invIndex, IIndexOperationContext opCtx) + public OnDiskInvertedIndexRangeSearchCursor(OnDiskInvertedIndex invIndex, IIndexOperationContext opCtx) throws HyracksDataException { - this.btree = ((OnDiskInvertedIndex) invIndex).getBTree(); + this.btree = invIndex.getBTree(); this.btreeAccessor = btree.createAccessor(NoOpIndexAccessParameters.INSTANCE); this.invIndex = invIndex; this.opCtx = opCtx; @@ -87,11 +86,7 @@ return true; } // The current inverted-list-range-search cursor is exhausted. - try { - invListRangeSearchCursor.unloadPages(); - } finally { - invListRangeSearchCursor.close(); - } + invListRangeSearchCursor.close(); isInvListCursorOpen = false; openInvListRangeSearchCursor(); return isInvListCursorOpen; @@ -109,14 +104,8 @@ @Override public void doDestroy() throws HyracksDataException { try { - if (isInvListCursorOpen) { - try { - invListRangeSearchCursor.unloadPages(); - } finally { - isInvListCursorOpen = false; - invListRangeSearchCursor.destroy(); - } - } + invListRangeSearchCursor.destroy(); + isInvListCursorOpen = false; } finally { btreeCursor.destroy(); } @@ -125,14 +114,8 @@ @Override public void doClose() throws HyracksDataException { try { - if (isInvListCursorOpen) { - try { - invListRangeSearchCursor.unloadPages(); - } finally { - invListRangeSearchCursor.close(); - } - isInvListCursorOpen = false; - } + invListRangeSearchCursor.close(); + isInvListCursorOpen = false; } finally { btreeCursor.close(); } @@ -148,7 +131,8 @@ if (btreeCursor.hasNext()) { btreeCursor.next(); tokenTuple.reset(btreeCursor.getTuple()); - invIndex.openInvertedListCursor(invListRangeSearchCursor, tokenTuple, opCtx); + invIndex.openInvertedListCursor(btreeCursor.getTuple(), invListRangeSearchCursor, + (OnDiskInvertedIndexOpContext) opCtx); invListRangeSearchCursor.prepareLoadPages(); invListRangeSearchCursor.loadPages(); concatTuple.reset(); diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java index eff4f5a..8c6b386 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java @@ -117,7 +117,7 @@ short numTokens = ShortPointable.getShort(btreeTuple.getFieldData(PARTITIONING_NUM_TOKENS_FIELD), btreeTuple.getFieldStart(PARTITIONING_NUM_TOKENS_FIELD)); InvertedListCursor invListCursor = partSearcher.getCachedInvertedListCursor(); - openInvertedListCursor(btreeTuple, invListCursor); + openInvertedListCursor(btreeTuple, invListCursor, ctx); invListPartitions.addInvertedListCursor(invListCursor, numTokens); tokenExists = true; } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java index ba54ed4..1e812b4 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java @@ -62,6 +62,7 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.PartitionedOnDiskInvertedIndexFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.util.trace.ITracer; public class InvertedIndexUtils { @@ -129,7 +130,7 @@ ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, - IMetadataPageManagerFactory pageManagerFactory) throws HyracksDataException { + IMetadataPageManagerFactory pageManagerFactory, ITracer tracer) throws HyracksDataException { BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory); @@ -166,7 +167,7 @@ filterManager, bloomFilterFalsePositiveRate, diskBufferCache, fileManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, - invertedIndexFieldsForNonBulkLoadOps, durable); + invertedIndexFieldsForNonBulkLoadOps, durable, tracer); } public static PartitionedLSMInvertedIndex createPartitionedLSMInvertedIndex(IIOManager ioManager, @@ -178,7 +179,7 @@ ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, - IPageManagerFactory pageManagerFactory) throws HyracksDataException { + IPageManagerFactory pageManagerFactory, ITracer tracer) throws HyracksDataException { BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory); @@ -215,6 +216,6 @@ filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, diskBufferCache, fileManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterFields, - filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable); + filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable, tracer); } } diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java index ae8713c..2435cc4 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java @@ -55,6 +55,9 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestUtils.HyracksTaskTestContext; import org.apache.hyracks.storage.common.IIndex; import org.apache.hyracks.storage.common.IIndexAccessParameters; +import org.apache.hyracks.util.trace.ITraceCategoryRegistry; +import org.apache.hyracks.util.trace.TraceCategoryRegistry; +import org.apache.hyracks.util.trace.Tracer; @SuppressWarnings("rawtypes") public class LSMInvertedIndexTestContext extends OrderedIndexTestContext { @@ -184,7 +187,9 @@ harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(), invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, - invertedIndexFieldsForNonBulkLoadOps, true, harness.getMetadataPageManagerFactory()); + invertedIndexFieldsForNonBulkLoadOps, true, harness.getMetadataPageManagerFactory(), + new Tracer(LSMInvertedIndexTestContext.class.getSimpleName(), + ITraceCategoryRegistry.CATEGORIES_ALL, new TraceCategoryRegistry())); break; } case PARTITIONED_LSM: { @@ -195,7 +200,9 @@ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(), invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, - invertedIndexFieldsForNonBulkLoadOps, true, harness.getMetadataPageManagerFactory()); + invertedIndexFieldsForNonBulkLoadOps, true, harness.getMetadataPageManagerFactory(), + new Tracer(LSMInvertedIndexTestContext.class.getSimpleName(), + ITraceCategoryRegistry.CATEGORIES_ALL, new TraceCategoryRegistry())); break; } default: { -- To view, visit https://asterix-gerrit.ics.uci.edu/2514 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I7bbf6fe2d332db0e0368a4ce0f1ab218ba6da81d Gerrit-PatchSet: 3 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Luo Chen <[email protected]> Gerrit-Reviewer: Taewoo Kim <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
