Yingyi Bu has submitted this change and it was merged. Change subject: ASTERIXDB-1938: Keep empty disk components when merge/flush ......................................................................
ASTERIXDB-1938: Keep empty disk components when merge/flush During flush/merge, it is possible that the new disk component is empty. However, the current LSMDiskComponentBulkLoader would clean up empty disk component. This patch fix this behavior by introducing an extra parameter cleanupEmptyComponent. Change-Id: Ic4f7ec41f56a2f9124920d67657f88160634f0e7 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1826 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> BAD: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Dmitry Lychagin <[email protected]> Reviewed-by: Yingyi Bu <[email protected]> --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentBulkLoader.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java 16 files changed, 100 insertions(+), 92 deletions(-) Approvals: Dmitry Lychagin: Looks good to me, but someone else must approve Yingyi Bu: Looks good to me, approved Jenkins: Verified; No violations found; No violations found; Verified diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java index a11b835..cdde692 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java @@ -462,7 +462,7 @@ } componentBulkLoader = - createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true); + createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true, true); } // It is expected that the mode was set to insert operation before diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java index abf3fa7..1c99b17 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java @@ -277,8 +277,8 @@ @Override public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, - boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter) - throws HyracksDataException { + boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter, + boolean cleanupEmptyComponent) throws HyracksDataException { BloomFilterSpecification bloomFilterSpec = null; if (numElementsHint > 0) { int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint); @@ -286,12 +286,13 @@ } if (withFilter && filterFields != null) { return new LSMBTreeWithBuddyDiskComponentBulkLoader((LSMBTreeWithBuddyDiskComponent) component, - bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, filterManager, - treeFields, filterFields, + bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, cleanupEmptyComponent, + filterManager, treeFields, filterFields, MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories())); } else { return new LSMBTreeWithBuddyDiskComponentBulkLoader((LSMBTreeWithBuddyDiskComponent) component, - bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex); + bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, + cleanupEmptyComponent); } } @@ -412,7 +413,8 @@ .getNumElements(); } - componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false); + componentBulkLoader = + createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false, false); try { while (buddyBtreeCursor.hasNext()) { @@ -424,7 +426,7 @@ buddyBtreeCursor.close(); } } else { - componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, 0L, false, false); + componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, 0L, false, false, false); } try { @@ -617,7 +619,7 @@ } componentBulkLoader = - createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true); + createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true, false); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java index f5f908f..e830b3e 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java @@ -326,7 +326,7 @@ createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getBloomFilterTarget(), true); ILSMDiskComponentBulkLoader componentBulkLoader = - createComponentBulkLoader(component, 1.0f, false, numElements, false, false); + createComponentBulkLoader(component, 1.0f, false, numElements, false, false, false); IIndexCursor scanCursor = accessor.createSearchCursor(false); accessor.search(scanCursor, nullPred); @@ -380,7 +380,7 @@ createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBloomFilterTarget(), true); ILSMDiskComponentBulkLoader componentBulkLoader = - createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false); + createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false, false); try { while (cursor.hasNext()) { cursor.next(); @@ -430,8 +430,8 @@ @Override public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, - boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter) - throws HyracksDataException { + boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter, + boolean cleanupEmptyComponent) throws HyracksDataException { BloomFilterSpecification bloomFilterSpec = null; if (hasBloomFilter) { int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint); @@ -440,11 +440,11 @@ if (withFilter && filterFields != null) { return new LSMBTreeDiskComponentBulkLoader((LSMBTreeDiskComponent) component, bloomFilterSpec, fillFactor, - verifyInput, numElementsHint, checkIfEmptyIndex, filterManager, treeFields, filterFields, - MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories())); + verifyInput, numElementsHint, checkIfEmptyIndex, cleanupEmptyComponent, filterManager, treeFields, + filterFields, MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories())); } else { return new LSMBTreeDiskComponentBulkLoader((LSMBTreeDiskComponent) component, bloomFilterSpec, fillFactor, - verifyInput, numElementsHint, checkIfEmptyIndex); + verifyInput, numElementsHint, checkIfEmptyIndex, cleanupEmptyComponent); } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java index 239eba2..c169d85 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java @@ -33,8 +33,8 @@ throws HyracksDataException { this.lsmIndex = lsmIndex; this.component = lsmIndex.createBulkLoadTarget(); - this.componentBulkLoader = - lsmIndex.createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true); + this.componentBulkLoader = lsmIndex.createComponentBulkLoader(component, fillFactor, verifyInput, + numElementsHint, false, true, true); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java index a5720de..43373b5 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java @@ -32,18 +32,18 @@ //with filter public LSMBTreeDiskComponentBulkLoader(LSMBTreeDiskComponent component, BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, - ILSMComponentFilterManager filterManager, int[] indexFields, int[] filterFields, MultiComparator filterCmp) - throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, filterManager, - indexFields, filterFields, filterCmp); + boolean cleanupEmptyComponent, ILSMComponentFilterManager filterManager, int[] indexFields, + int[] filterFields, MultiComparator filterCmp) throws HyracksDataException { + super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, + cleanupEmptyComponent, filterManager, indexFields, filterFields, filterCmp); } //without filter public LSMBTreeDiskComponentBulkLoader(LSMBTreeDiskComponent component, BloomFilterSpecification bloomFilterSpec, - float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex) - throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, null, null, null, - null); + float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, + boolean cleanupEmptyComponent) throws HyracksDataException { + super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, + cleanupEmptyComponent, null, null, null, null); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java index e4ab6d4..b3e7c6c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java @@ -33,18 +33,18 @@ //with filter public LSMBTreeWithBuddyDiskComponentBulkLoader(LSMBTreeWithBuddyDiskComponent component, BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint, - boolean checkIfEmptyIndex, ILSMComponentFilterManager filterManager, int[] indexFields, int[] filterFields, - MultiComparator filterCmp) throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, filterManager, - indexFields, filterFields, filterCmp); + boolean checkIfEmptyIndex, boolean cleanupEmptyComponent, ILSMComponentFilterManager filterManager, + int[] indexFields, int[] filterFields, MultiComparator filterCmp) throws HyracksDataException { + super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, + cleanupEmptyComponent, filterManager, indexFields, filterFields, filterCmp); } //without filter public LSMBTreeWithBuddyDiskComponentBulkLoader(LSMBTreeWithBuddyDiskComponent component, BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint, - boolean checkIfEmptyIndex) throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, null, null, null, - null); + boolean checkIfEmptyIndex, boolean cleanupEmptyComponent) throws HyracksDataException { + super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, + cleanupEmptyComponent, null, null, null, null); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java index c7032ef..9e1f30d 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java @@ -151,11 +151,12 @@ * @param numElementsHint * @param checkIfEmptyIndex * @param withFilter + * @param cleanupEmptyComponent * @return * @throws HyracksDataException */ ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, - boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter) - throws HyracksDataException; + boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter, + boolean cleanupEmptyComponent) throws HyracksDataException; } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java index ad47bbc..351f619 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentBulkLoader.java @@ -43,6 +43,7 @@ protected final PermutingTupleReference indexTuple; protected final PermutingTupleReference filterTuple; protected final MultiComparator filterCmp; + protected final boolean cleanupEmptyComponent; protected boolean cleanedUpArtifacts = false; protected boolean isEmptyComponent = true; @@ -51,8 +52,8 @@ //with filter public AbstractLSMDiskComponentBulkLoader(ILSMDiskComponent component, BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, - ILSMComponentFilterManager filterManager, int[] indexFields, int[] filterFields, MultiComparator filterCmp) - throws HyracksDataException { + boolean cleanupEmptyComponent, ILSMComponentFilterManager filterManager, int[] indexFields, + int[] filterFields, MultiComparator filterCmp) throws HyracksDataException { this.component = component; this.indexBulkLoader = getIndex(component).createBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex); @@ -62,6 +63,7 @@ } else { this.bloomFilterBuilder = null; } + this.cleanupEmptyComponent = cleanupEmptyComponent; if (filterManager != null) { this.filterManager = filterManager; this.indexTuple = new PermutingTupleReference(indexFields); @@ -154,7 +156,7 @@ } indexBulkLoader.end(); - if (isEmptyComponent) { + if (isEmptyComponent && cleanupEmptyComponent) { cleanupArtifacts(); } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java index d3eafdc6..fc7310c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMDiskComponentWithBuddyBulkLoader.java @@ -35,10 +35,10 @@ //with filter public AbstractLSMDiskComponentWithBuddyBulkLoader(ILSMDiskComponent component, BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint, - boolean checkIfEmptyIndex, ILSMComponentFilterManager filterManager, int[] indexFields, int[] filterFields, - MultiComparator filterCmp) throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, filterManager, - indexFields, filterFields, filterCmp); + boolean checkIfEmptyIndex, boolean cleanupEmptyComponent, ILSMComponentFilterManager filterManager, + int[] indexFields, int[] filterFields, MultiComparator filterCmp) throws HyracksDataException { + super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, + cleanupEmptyComponent, filterManager, indexFields, filterFields, filterCmp); // BuddyBTree must be created even if it could be empty, // since without it the component is not considered as valid. @@ -124,7 +124,7 @@ indexBulkLoader.end(); buddyBTreeBulkLoader.end(); - if (isEmptyComponent) { + if (isEmptyComponent && cleanupEmptyComponent) { cleanupArtifacts(); } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java index bd4797a..d5edbb5 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java @@ -364,7 +364,7 @@ } ILSMDiskComponentBulkLoader componentBulkLoader = - createComponentBulkLoader(component, 1.0f, false, numBTreeTuples, false, false); + createComponentBulkLoader(component, 1.0f, false, numBTreeTuples, false, false, false); // Create a scan cursor on the deleted keys BTree underlying the in-memory inverted index. IIndexCursor deletedKeysScanCursor = deletedKeysBTreeAccessor.createSearchCursor(false); @@ -444,7 +444,7 @@ .getNumElements(); } - componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numElements, false, false); + componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numElements, false, false, false); try { while (btreeCursor.hasNext()) { btreeCursor.next(); @@ -455,7 +455,7 @@ btreeCursor.close(); } } else { - componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, 0L, false, false); + componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, 0L, false, false, false); } try { @@ -491,8 +491,8 @@ @Override public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, - boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter) - throws HyracksDataException { + boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter, + boolean cleanupEmptyComponent) throws HyracksDataException { BloomFilterSpecification bloomFilterSpec = null; if (numElementsHint > 0) { int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint); @@ -500,12 +500,13 @@ } if (withFilter && filterFields != null) { return new LSMInvertedIndexDiskComponentBulkLoader((LSMInvertedIndexDiskComponent) component, - bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, filterManager, - treeFields, filterFields, + bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, cleanupEmptyComponent, + filterManager, treeFields, filterFields, MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories())); } else { return new LSMInvertedIndexDiskComponentBulkLoader((LSMInvertedIndexDiskComponent) component, - bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex); + bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, + cleanupEmptyComponent); } } @@ -526,7 +527,7 @@ component = createBulkLoadTarget(); componentBulkLoader = - createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true); + createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true, true); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java index 38e1c42..daa93ae 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentBulkLoader.java @@ -34,18 +34,18 @@ //with filter public LSMInvertedIndexDiskComponentBulkLoader(LSMInvertedIndexDiskComponent component, BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint, - boolean checkIfEmptyIndex, ILSMComponentFilterManager filterManager, int[] indexFields, int[] filterFields, - MultiComparator filterCmp) throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, filterManager, - indexFields, filterFields, filterCmp); + boolean checkIfEmptyIndex, boolean cleanupEmptyComponent, ILSMComponentFilterManager filterManager, + int[] indexFields, int[] filterFields, MultiComparator filterCmp) throws HyracksDataException { + super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, + cleanupEmptyComponent, filterManager, indexFields, filterFields, filterCmp); } //without filter public LSMInvertedIndexDiskComponentBulkLoader(LSMInvertedIndexDiskComponent component, BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint, - boolean checkIfEmptyIndex) throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, null, null, null, - null); + boolean checkIfEmptyIndex, boolean cleanupEmptyComponent) throws HyracksDataException { + super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, + cleanupEmptyComponent, null, null, null, null); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java index eb838a9..56af16d 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java @@ -197,7 +197,7 @@ } ILSMDiskComponentBulkLoader componentBulkLoader = - createComponentBulkLoader(component, 1.0f, false, numBTreeTuples, false, false); + createComponentBulkLoader(component, 1.0f, false, numBTreeTuples, false, false, false); ITreeIndexCursor cursor; IBinaryComparatorFactory[] linearizerArray = { linearizer }; @@ -286,7 +286,8 @@ numElements += ((LSMRTreeDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter() .getNumElements(); } - componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false); + componentBulkLoader = + createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false, false); LSMRTreeDeletedKeysBTreeMergeCursor btreeCursor = new LSMRTreeDeletedKeysBTreeMergeCursor(opCtx); search(opCtx, btreeCursor, rtreeSearchPred); @@ -301,7 +302,7 @@ } } else { //no buddy-btree needed - componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, 0L, false, false); + componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, 0L, false, false, false); } //search old rtree components @@ -345,8 +346,8 @@ @Override public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, - boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter) - throws HyracksDataException { + boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter, + boolean cleanupEmptyComponent) throws HyracksDataException { BloomFilterSpecification bloomFilterSpec = null; if (numElementsHint > 0) { int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint); @@ -354,11 +355,11 @@ } if (withFilter && filterFields != null) { return new LSMRTreeDiskComponentBulkLoader((LSMRTreeDiskComponent) component, bloomFilterSpec, fillFactor, - verifyInput, numElementsHint, checkIfEmptyIndex, filterManager, treeFields, filterFields, - MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories())); + verifyInput, numElementsHint, checkIfEmptyIndex, cleanupEmptyComponent, filterManager, treeFields, + filterFields, MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories())); } else { return new LSMRTreeDiskComponentBulkLoader((LSMRTreeDiskComponent) component, bloomFilterSpec, fillFactor, - verifyInput, numElementsHint, checkIfEmptyIndex); + verifyInput, numElementsHint, checkIfEmptyIndex, cleanupEmptyComponent); } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java index fbdd37a..b30e2fa 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java @@ -35,8 +35,8 @@ // Note that by using a flush target file name, we state that the // new bulk loaded tree is "newer" than any other merged tree. this.component = lsmIndex.createBulkLoadTarget(); - this.componentBulkLoader = - lsmIndex.createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true); + this.componentBulkLoader = lsmIndex.createComponentBulkLoader(component, fillFactor, verifyInput, + numElementsHint, false, true, true); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentBulkLoader.java index ff0a299..e72918a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentBulkLoader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentBulkLoader.java @@ -33,18 +33,18 @@ //with filter public LSMRTreeDiskComponentBulkLoader(LSMRTreeDiskComponent component, BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, - ILSMComponentFilterManager filterManager, int[] indexFields, int[] filterFields, MultiComparator filterCmp) - throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, filterManager, - indexFields, filterFields, filterCmp); + boolean cleanupEmptyComponent, ILSMComponentFilterManager filterManager, int[] indexFields, + int[] filterFields, MultiComparator filterCmp) throws HyracksDataException { + super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, + cleanupEmptyComponent, filterManager, indexFields, filterFields, filterCmp); } //without filter public LSMRTreeDiskComponentBulkLoader(LSMRTreeDiskComponent component, BloomFilterSpecification bloomFilterSpec, - float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex) - throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, null, null, null, - null); + float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, + boolean cleanupEmptyComponent) throws HyracksDataException { + super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, + cleanupEmptyComponent, null, null, null, null); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java index 81fd981..ae72884 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java @@ -137,7 +137,7 @@ memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate); LSMRTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), null, null, true); ILSMDiskComponentBulkLoader componentBulkLoader = - createComponentBulkLoader(component, 1.0f, false, 0L, false, false); + createComponentBulkLoader(component, 1.0f, false, 0L, false, false, false); // Since the LSM-RTree is used as a secondary assumption, the // primary key will be the last comparator in the BTree comparators @@ -224,7 +224,7 @@ LSMRTreeDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(), null, null, true); ILSMDiskComponentBulkLoader componentBulkLoader = - createComponentBulkLoader(component, 1.0f, false, 0L, false, false); + createComponentBulkLoader(component, 1.0f, false, 0L, false, false, false); try { while (cursor.hasNext()) { cursor.next(); @@ -258,15 +258,16 @@ @Override public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, - boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter) - throws HyracksDataException { + boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter, + boolean cleanupEmptyComponent) throws HyracksDataException { if (withFilter && filterFields != null) { return new LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader((LSMRTreeDiskComponent) component, null, - fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, filterManager, treeFields, - filterFields, MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories())); + fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, cleanupEmptyComponent, filterManager, + treeFields, filterFields, + MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories())); } else { return new LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader((LSMRTreeDiskComponent) component, null, - fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex); + fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, cleanupEmptyComponent); } } @@ -285,7 +286,7 @@ component = createBulkLoadTarget(); componentBulkLoader = - createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true); + createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true, true); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java index 88a2e1e..22046aa 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java @@ -32,18 +32,18 @@ //with filter public LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader(LSMRTreeDiskComponent component, BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint, - boolean checkIfEmptyIndex, ILSMComponentFilterManager filterManager, int[] indexFields, int[] filterFields, - MultiComparator filterCmp) throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, filterManager, - indexFields, filterFields, filterCmp); + boolean checkIfEmptyIndex, boolean cleanupEmptyComponent, ILSMComponentFilterManager filterManager, + int[] indexFields, int[] filterFields, MultiComparator filterCmp) throws HyracksDataException { + super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, + cleanupEmptyComponent, filterManager, indexFields, filterFields, filterCmp); } //without filter public LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader(LSMRTreeDiskComponent component, BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint, - boolean checkIfEmptyIndex) throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, null, null, null, - null); + boolean checkIfEmptyIndex, boolean cleanupEmptyComponent) throws HyracksDataException { + super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, + cleanupEmptyComponent, null, null, null, null); } @Override -- To view, visit https://asterix-gerrit.ics.uci.edu/1826 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ic4f7ec41f56a2f9124920d67657f88160634f0e7 Gerrit-PatchSet: 3 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen <[email protected]> Gerrit-Reviewer: Dmitry Lychagin <[email protected]> Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: Yingyi Bu <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
