[NO ISSUE][STO][IDX] LSM storage cleanup - user model changes: no - storage format changes: no - interface changes: yes
- Replaced component lifecycle-related fabric methods in AbstractLSMIndex with direct method calls of lsmComponent functions. - Extracted common lifecycle-related functionality from index-specific disk/memory lsmComponents to interfaces. - Introduced composable disk component bulkloader design which assembles the proper bulkload pipeline from individual elements populating lsmFilters, bloomFilters, buddyBTrees\deletedKeysBTrees, bTress\rTrees\invIndexes. - Changed methods to return index-specific versions of objects (accessors, components, index instances) to avoid nasty downcasting. Change-Id: I6739d751b990e7a28e03e32a5de6e2b670d37a1e Reviewed-on: https://asterix-gerrit.ics.uci.edu/2014 Sonar-Qube: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Contrib: Jenkins <[email protected]> Reviewed-by: abdullah alamoudi <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/77f89525 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/77f89525 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/77f89525 Branch: refs/heads/master Commit: 77f8952573443eaf81270f509ffd39fa84115411 Parents: 536aa3f Author: Ildar Absalyamov <[email protected]> Authored: Fri Oct 13 12:22:52 2017 -0700 Committer: Ildar Absalyamov <[email protected]> Committed: Sun Oct 15 23:06:51 2017 -0700 ---------------------------------------------------------------------- asterixdb/asterix-common/pom.xml | 10 + .../LSMBTreeIOOperationCallback.java | 5 +- .../LSMBTreeWithBuddyIOOperationCallback.java | 5 +- .../LSMInvertedIndexIOOperationCallback.java | 2 +- .../LSMRTreeIOOperationCallback.java | 5 +- .../LSMSecondaryIndexBulkLoadNodePushable.java | 4 +- .../hyracks/storage/am/btree/impls/BTree.java | 9 +- .../am/common/impls/AbstractTreeIndex.java | 1 + .../am/lsm/btree/impls/ExternalBTree.java | 97 +++---- .../lsm/btree/impls/ExternalBTreeWithBuddy.java | 227 +++------------ .../storage/am/lsm/btree/impls/LSMBTree.java | 242 +++------------- .../am/lsm/btree/impls/LSMBTreeBulkLoader.java | 62 ----- .../lsm/btree/impls/LSMBTreeDiskComponent.java | 68 ++--- .../impls/LSMBTreeDiskComponentBulkLoader.java | 59 ---- .../impls/LSMBTreeDiskComponentFactory.java | 21 +- .../impls/LSMBTreeDiskComponentScanCursor.java | 6 +- .../btree/impls/LSMBTreeMemoryComponent.java | 17 +- .../am/lsm/btree/impls/LSMBTreeOpContext.java | 14 +- .../btree/impls/LSMBTreePointSearchCursor.java | 12 +- .../btree/impls/LSMBTreeRangeSearchCursor.java | 8 +- .../LSMBTreeWithBloomFilterDiskComponent.java | 86 ++++++ ...TreeWithBloomFilterDiskComponentFactory.java | 54 ++++ .../impls/LSMBTreeWithBuddyAbstractCursor.java | 19 +- .../impls/LSMBTreeWithBuddyDiskComponent.java | 76 +++--- ...SMBTreeWithBuddyDiskComponentBulkLoader.java | 65 ----- .../LSMBTreeWithBuddyDiskComponentFactory.java | 28 +- .../impls/LSMBTreeWithBuddyMemoryComponent.java | 26 +- .../btree/impls/LSMBuddyBTreeMergeCursor.java | 2 +- .../am/lsm/btree/utils/LSMBTreeUtil.java | 51 +++- ...AbstractLSMWithBloomFilterDiskComponent.java | 105 +++++++ .../api/AbstractLSMWithBuddyDiskComponent.java | 96 +++++++ .../AbstractLSMWithBuddyMemoryComponent.java | 61 +++++ .../am/lsm/common/api/ILSMComponent.java | 6 + .../am/lsm/common/api/ILSMDiskComponent.java | 110 +++++++- .../common/api/ILSMDiskComponentBulkLoader.java | 7 + .../common/api/ILSMDiskComponentFactory.java | 4 +- .../storage/am/lsm/common/api/ILSMIndex.java | 17 -- .../am/lsm/common/api/ILSMMemoryComponent.java | 29 +- .../common/impls/AbstractLSMDiskComponent.java | 89 +++++- .../AbstractLSMDiskComponentBulkLoader.java | 202 -------------- ...ractLSMDiskComponentWithBuddyBulkLoader.java | 156 ----------- .../am/lsm/common/impls/AbstractLSMIndex.java | 126 +++++---- .../impls/AbstractLSMMemoryComponent.java | 46 +++- .../lsm/common/impls/BloomFilterBulkLoader.java | 67 +++++ .../ChainedLSMDiskComponentBulkLoader.java | 110 ++++++++ .../am/lsm/common/impls/EmptyComponent.java | 67 +++++ .../lsm/common/impls/ExternalIndexHarness.java | 4 +- .../am/lsm/common/impls/FilterBulkLoader.java | 82 ++++++ .../impls/IChainedComponentBulkLoader.java | 63 +++++ .../common/impls/IndexWithBuddyBulkLoader.java | 72 +++++ .../storage/am/lsm/common/impls/LSMHarness.java | 4 +- .../am/lsm/common/impls/LSMIndexBulkLoader.java | 67 +++++ .../impls/LSMIndexDiskComponentBulkLoader.java | 64 +++++ .../api/IInPlaceInvertedIndex.java | 9 - .../invertedindex/impls/LSMInvertedIndex.java | 273 +++---------------- .../impls/LSMInvertedIndexDiskComponent.java | 64 +++-- ...LSMInvertedIndexDiskComponentBulkLoader.java | 71 ----- .../LSMInvertedIndexDiskComponentFactory.java | 6 +- .../impls/LSMInvertedIndexMemoryComponent.java | 32 +-- .../impls/LSMInvertedIndexOpContext.java | 14 +- .../impls/PartitionedLSMInvertedIndex.java | 32 +-- .../inmemory/InMemoryInvertedIndex.java | 6 +- .../inmemory/InMemoryInvertedIndexAccessor.java | 6 +- .../InMemoryInvertedIndexOpContext.java | 4 +- .../PartitionedInMemoryInvertedIndex.java | 6 +- .../ondisk/OnDiskInvertedIndex.java | 4 +- .../ondisk/OnDiskInvertedIndexFactory.java | 3 +- .../ondisk/PartitionedOnDiskInvertedIndex.java | 3 +- .../PartitionedOnDiskInvertedIndexFactory.java | 3 +- .../invertedindex/util/InvertedIndexUtils.java | 26 +- .../am/lsm/rtree/impls/AbstractLSMRTree.java | 117 ++------ .../am/lsm/rtree/impls/ExternalRTree.java | 111 +++----- .../storage/am/lsm/rtree/impls/LSMRTree.java | 180 +++--------- .../lsm/rtree/impls/LSMRTreeAbstractCursor.java | 16 +- .../am/lsm/rtree/impls/LSMRTreeBulkLoader.java | 60 ---- .../LSMRTreeDeletedKeysBTreeMergeCursor.java | 2 +- .../lsm/rtree/impls/LSMRTreeDiskComponent.java | 89 +++--- .../impls/LSMRTreeDiskComponentBulkLoader.java | 65 ----- .../impls/LSMRTreeDiskComponentFactory.java | 6 +- .../rtree/impls/LSMRTreeMemoryComponent.java | 24 +- .../am/lsm/rtree/impls/LSMRTreeOpContext.java | 12 +- .../impls/LSMRTreeWithAntiMatterTuples.java | 155 ++--------- ...AntiMatterTuplesDiskComponentBulkLoader.java | 59 ---- ...ithAntiMatterTuplesDiskComponentFactory.java | 9 +- ...SMRTreeWithAntiMatterTuplesSearchCursor.java | 6 +- .../LSMRTreeWithAntimatterDiskComponent.java | 71 +++++ .../am/lsm/rtree/utils/LSMRTreeUtils.java | 43 ++- .../hyracks/storage/am/rtree/impls/RTree.java | 6 +- .../apache/hyracks/storage/common/IIndex.java | 11 + .../storage/am/common/IndexTestContext.java | 3 +- .../am/lsm/btree/LSMBTreeFileManagerTest.java | 7 +- ...TreeUpdateInPlaceScanDiskComponentsTest.java | 12 +- .../storage/am/lsm/btree/impl/TestLsmBtree.java | 13 +- .../am/lsm/btree/impl/TestLsmBtreeUtil.java | 21 +- 94 files changed, 2156 insertions(+), 2439 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/asterixdb/asterix-common/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/pom.xml b/asterixdb/asterix-common/pom.xml index 6df7049..42a8004 100644 --- a/asterixdb/asterix-common/pom.xml +++ b/asterixdb/asterix-common/pom.xml @@ -168,6 +168,16 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <configuration> + <ignoredUnusedDeclaredDependencies> + <ignoredUnusedDeclaredDependency>org.apache.hyracks:hyracks-storage-am-lsm-btree:*</ignoredUnusedDeclaredDependency> + <ignoredUnusedDeclaredDependency>org.apache.hyracks:hyracks-storage-am-lsm-rtree:*</ignoredUnusedDeclaredDependency> + </ignoredUnusedDeclaredDependencies> + </configuration> + </plugin> </plugins> </build> <dependencies> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java index 8c91879..147bc10 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java @@ -20,8 +20,8 @@ package org.apache.asterix.common.ioopcallbacks; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.common.api.IMetadataPageManager; -import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeDiskComponent; import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; @@ -35,9 +35,8 @@ public class LSMBTreeIOOperationCallback extends AbstractLSMIOOperationCallback public long getComponentFileLSNOffset(ILSMDiskComponent diskComponent, String diskComponentFilePath) throws HyracksDataException { if (diskComponentFilePath.endsWith(LSMBTreeFileManager.BTREE_SUFFIX)) { - LSMBTreeDiskComponent btreeComponent = (LSMBTreeDiskComponent) diskComponent; IMetadataPageManager metadataPageManager = - (IMetadataPageManager) btreeComponent.getBTree().getPageManager(); + (IMetadataPageManager) ((BTree) diskComponent.getIndex()).getPageManager(); return metadataPageManager.getFileOffset(metadataPageManager.createMetadataFrame(), LSN_KEY); } return INVALID; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java index 9f50f57..5b6de6c 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java @@ -19,8 +19,8 @@ package org.apache.asterix.common.ioopcallbacks; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.common.api.IMetadataPageManager; -import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyDiskComponent; import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyFileManager; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; @@ -30,9 +30,8 @@ public class LSMBTreeWithBuddyIOOperationCallback extends AbstractLSMIOOperation public long getComponentFileLSNOffset(ILSMDiskComponent diskComponent, String diskComponentFilePath) throws HyracksDataException { if (diskComponentFilePath.endsWith(LSMBTreeWithBuddyFileManager.BTREE_SUFFIX)) { - LSMBTreeWithBuddyDiskComponent btreeComponent = (LSMBTreeWithBuddyDiskComponent) diskComponent; IMetadataPageManager metadataPageManager = - (IMetadataPageManager) btreeComponent.getBTree().getPageManager(); + (IMetadataPageManager) ((BTree) diskComponent.getIndex()).getPageManager(); return metadataPageManager.getFileOffset(metadataPageManager.createMetadataFrame(), LSN_KEY); } return INVALID; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java index 69286d5..3c91a6a 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java @@ -37,7 +37,7 @@ public class LSMInvertedIndexIOOperationCallback extends AbstractLSMIOOperationC if (diskComponentFilePath.endsWith(LSMInvertedIndexFileManager.DELETED_KEYS_BTREE_SUFFIX)) { LSMInvertedIndexDiskComponent invIndexComponent = (LSMInvertedIndexDiskComponent) diskComponent; IMetadataPageManager metadataPageManager = - (IMetadataPageManager) invIndexComponent.getDeletedKeysBTree().getPageManager(); + (IMetadataPageManager) invIndexComponent.getBuddyIndex().getPageManager(); return metadataPageManager.getFileOffset(metadataPageManager.createMetadataFrame(), LSN_KEY); } return INVALID; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java index 1001af9..063fe38 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java @@ -22,8 +22,8 @@ package org.apache.asterix.common.ioopcallbacks; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.common.api.IMetadataPageManager; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; -import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeDiskComponent; import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeFileManager; +import org.apache.hyracks.storage.am.rtree.impls.RTree; public class LSMRTreeIOOperationCallback extends AbstractLSMIOOperationCallback { @@ -35,9 +35,8 @@ public class LSMRTreeIOOperationCallback extends AbstractLSMIOOperationCallback public long getComponentFileLSNOffset(ILSMDiskComponent diskComponent, String diskComponentFilePath) throws HyracksDataException { if (diskComponentFilePath.endsWith(LSMRTreeFileManager.RTREE_SUFFIX)) { - LSMRTreeDiskComponent rtreeComponent = (LSMRTreeDiskComponent) diskComponent; IMetadataPageManager metadataPageManager = - (IMetadataPageManager) rtreeComponent.getRTree().getPageManager(); + (IMetadataPageManager) ((RTree) diskComponent.getIndex()).getPageManager(); return metadataPageManager.getFileOffset(metadataPageManager.createMetadataFrame(), LSN_KEY); } return INVALID; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadNodePushable.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadNodePushable.java index 12f49a8..cb81b64 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadNodePushable.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadNodePushable.java @@ -199,8 +199,8 @@ public class LSMSecondaryIndexBulkLoadNodePushable extends AbstractLSMSecondaryI endCurrentComponent(); component = secondaryIndex.createBulkLoadTarget(); - componentBulkLoader = secondaryIndex.createComponentBulkLoader(component, 1.0f, false, - getNumDeletedTuples(componentPos), false, true, true); + int numTuples = getNumDeletedTuples(componentPos); + componentBulkLoader = component.createBulkLoader(1.0f, false, numTuples, false, true, true); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java index 7206fee..5e3042e 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java @@ -118,8 +118,7 @@ public class BTree extends AbstractTreeIndex { // Stack validation protocol: // * parent pushes the validation information onto the stack before validation // * child pops the validation information off of the stack after validating - BTreeAccessor accessor = - (BTreeAccessor) createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); + BTreeAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); PageValidationInfo pvi = accessor.ctx.createPageValidationInfo(null); accessor.ctx.getValidationInfos().addFirst(pvi); if (isActive) { @@ -819,12 +818,12 @@ public class BTree extends AbstractTreeIndex { } @Override - public ITreeIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, + public BTreeAccessor createAccessor(IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) { return new BTreeAccessor(this, modificationCallback, searchCallback); } - public ITreeIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, + public BTreeAccessor createAccessor(IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback, int[] logTupleFields) { return new BTreeAccessor(this, modificationCallback, searchCallback, logTupleFields); } @@ -892,7 +891,7 @@ public class BTree extends AbstractTreeIndex { } @Override - public ITreeIndexCursor createSearchCursor(boolean exclusive) { + public BTreeRangeSearchCursor createSearchCursor(boolean exclusive) { IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame(); return new BTreeRangeSearchCursor(leafFrame, exclusive); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java index 8ffd0eb..ae66402 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java @@ -121,6 +121,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex { isActive = false; } + @Override public void purge() throws HyracksDataException { if (isActive) { throw HyracksDataException.create(ErrorCode.CANNOT_PURGE_ACTIVE_INDEX); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java index 3775985..24f513a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java @@ -28,8 +28,6 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory; import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.common.api.IIndexOperationContext; import org.apache.hyracks.storage.am.common.api.IMetadataPageManager; @@ -41,6 +39,7 @@ import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; @@ -56,12 +55,12 @@ import org.apache.hyracks.storage.am.lsm.common.impls.ExternalIndexHarness; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor; import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor.ICursorFactory; -import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; import org.apache.hyracks.storage.common.IIndexBulkLoader; import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.ISearchPredicate; +import org.apache.hyracks.storage.common.buffercache.IBufferCache; /** * This is an lsm b-tree that does not have memory component and is modified @@ -76,7 +75,7 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { // This component factory has to be different since it uses different tuple // writer in it's leaf frames to support inserting both // regular and delete tuples - private final LSMBTreeDiskComponentFactory transactionComponentFactory; + private final ILSMDiskComponentFactory transactionComponentFactory; // A second disk component list that will be used when a transaction is // committed and will be seen by subsequent accessors private final List<ILSMDiskComponent> secondDiskComponents; @@ -90,16 +89,15 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { //TODO remove BloomFilter from external dataset's secondary LSMBTree index public ExternalBTree(IIOManager ioManager, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory, - ILSMIndexFileManager fileManager, TreeIndexFactory<BTree> diskBTreeFactory, - TreeIndexFactory<BTree> bulkLoadBTreeFactory, BloomFilterFactory bloomFilterFactory, + IBufferCache bufferCache, ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory, + ILSMDiskComponentFactory bulkLoadComponentFactory, ILSMDiskComponentFactory transactionComponentFactory, double bloomFilterFalsePositiveRate, IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, - TreeIndexFactory<BTree> transactionBTreeFactory, boolean durable) { - super(ioManager, insertLeafFrameFactory, deleteLeafFrameFactory, fileManager, diskBTreeFactory, - bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, cmpFactories, mergePolicy, - opTracker, ioScheduler, ioOpCallback, false, durable); - this.transactionComponentFactory = - new LSMBTreeDiskComponentFactory(transactionBTreeFactory, bloomFilterFactory, null); + boolean durable) { + super(ioManager, insertLeafFrameFactory, deleteLeafFrameFactory, bufferCache, fileManager, componentFactory, + bulkLoadComponentFactory, bloomFilterFalsePositiveRate, cmpFactories, mergePolicy, opTracker, + ioScheduler, ioOpCallback, false, durable); + this.transactionComponentFactory = transactionComponentFactory; this.secondDiskComponents = new LinkedList<>(); this.interiorFrameFactory = interiorFrameFactory; } @@ -164,7 +162,8 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { // This method creates the appropriate opContext for the targeted version public ExternalBTreeOpContext createOpContext(ISearchOperationCallback searchCallback, int targetVersion) { return new ExternalBTreeOpContext(insertLeafFrameFactory, deleteLeafFrameFactory, searchCallback, - componentFactory.getBloomFilterKeyFields().length, cmpFactories, targetVersion, getLsmHarness()); + ((LSMBTreeWithBloomFilterDiskComponentFactory) componentFactory).getBloomFilterKeyFields().length, + cmpFactories, targetVersion, getLsmHarness()); } // The only reason to override the following method is that it uses a different context object @@ -188,8 +187,8 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { } } ITreeIndexCursor cursor = new LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples); - BTree firstBTree = ((LSMBTreeDiskComponent) mergingComponents.get(0)).getBTree(); - BTree lastBTree = ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1)).getBTree(); + BTree firstBTree = ((LSMBTreeDiskComponent) mergingComponents.get(0)).getIndex(); + BTree lastBTree = ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1)).getIndex(); FileReference firstFile = firstBTree.getFileReference(); FileReference lastFile = lastBTree.getFileReference(); LSMComponentFileReferences relMergeFileRefs = @@ -252,32 +251,23 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { //First time activation List<LSMComponentFileReferences> validFileReferences; validFileReferences = fileManager.cleanupAndGetValidFiles(); - for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) { - LSMBTreeDiskComponent component; - component = - createDiskComponent(componentFactory, lsmComonentFileReference.getInsertIndexFileReference(), - lsmComonentFileReference.getBloomFilterFileReference(), false); + for (LSMComponentFileReferences lsmComponentFileReferences : validFileReferences) { + ILSMDiskComponent component = + createDiskComponent(componentFactory, lsmComponentFileReferences.getInsertIndexFileReference(), + null, lsmComponentFileReferences.getBloomFilterFileReference(), false); diskComponents.add(component); secondDiskComponents.add(component); } getLsmHarness().indexFirstTimeActivated(); } else { // This index has been opened before - for (ILSMComponent c : diskComponents) { - LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; - BTree btree = component.getBTree(); - BloomFilter bloomFilter = component.getBloomFilter(); - btree.activate(); - bloomFilter.activate(); + for (ILSMDiskComponent c : diskComponents) { + c.activate(false); } - for (ILSMComponent c : secondDiskComponents) { + for (ILSMDiskComponent c : secondDiskComponents) { // Only activate non shared components if (!diskComponents.contains(c)) { - LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; - BTree btree = component.getBTree(); - BloomFilter bloomFilter = component.getBloomFilter(); - btree.activate(); - bloomFilter.activate(); + c.activate(false); } } } @@ -300,12 +290,12 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { cb.afterFinalize(LSMOperationType.FLUSH, null); } for (ILSMDiskComponent c : diskComponents) { - deactivateDiskComponent(c); + c.deactivateAndPurge(); } for (ILSMDiskComponent c : secondDiskComponents) { // Only deactivate non shared components if (!diskComponents.contains(c)) { - deactivateDiskComponent(c); + c.deactivateAndPurge(); } } isActive = false; @@ -320,22 +310,14 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { } getLsmHarness().indexClear(); - for (ILSMComponent c : diskComponents) { - LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; - component.getBloomFilter().deactivate(); - component.getBTree().deactivate(); - component.getBloomFilter().destroy(); - component.getBTree().destroy(); + for (ILSMDiskComponent c : diskComponents) { + c.deactivateAndDestroy(); // Remove from second list to avoid destroying twice secondDiskComponents.remove(c); } - for (ILSMComponent c : secondDiskComponents) { - LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; - component.getBloomFilter().deactivate(); - component.getBTree().deactivate(); - component.getBloomFilter().destroy(); - component.getBTree().destroy(); + for (ILSMDiskComponent c : secondDiskComponents) { + c.deactivateAndDestroy(); } diskComponents.clear(); secondDiskComponents.clear(); @@ -348,12 +330,12 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { throw new HyracksDataException("Failed to destroy the index since it is activated."); } for (ILSMDiskComponent c : diskComponents) { - destroyDiskComponent(c); + c.destroy(); // Remove from second list to avoid destroying twice secondDiskComponents.remove(c); } for (ILSMDiskComponent c : secondDiskComponents) { - destroyDiskComponent(c); + c.destroy(); } diskComponents.clear(); secondDiskComponents.clear(); @@ -448,7 +430,7 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { } componentBulkLoader = - createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true, true); + component.createBulkLoader(fillFactor, verifyInput, numElementsHint, false, true, true); } // It is expected that the mode was set to insert operation before @@ -466,10 +448,7 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { // Since this is a transaction component, validate and // deactivate. it could later be added or deleted component.markAsValid(durable); - BTree btree = ((LSMBTreeDiskComponent) component).getBTree(); - BloomFilter bloomFilter = ((LSMBTreeDiskComponent) component).getBloomFilter(); - btree.deactivate(); - bloomFilter.deactivate(); + component.deactivate(); } else { getLsmHarness().addBulkLoadedComponent(component); } @@ -502,7 +481,7 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { throw HyracksDataException.create(e); } return createDiskComponent(transactionComponentFactory, componentFileRefs.getInsertIndexFileReference(), - componentFileRefs.getBloomFilterFileReference(), true); + null, componentFileRefs.getBloomFilterFileReference(), true); } } @@ -568,11 +547,11 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { @Override public void commitTransaction() throws HyracksDataException { - LSMComponentFileReferences componentFileRefrences = fileManager.getTransactionFileReferenceForCommit(); - LSMBTreeDiskComponent component = null; - if (componentFileRefrences != null) { - component = createDiskComponent(componentFactory, componentFileRefrences.getInsertIndexFileReference(), - componentFileRefrences.getBloomFilterFileReference(), false); + LSMComponentFileReferences componentFileReferences = fileManager.getTransactionFileReferenceForCommit(); + ILSMDiskComponent component = null; + if (componentFileReferences != null) { + component = createDiskComponent(componentFactory, componentFileReferences.getInsertIndexFileReference(), + null, componentFileReferences.getBloomFilterFileReference(), false); } getLsmHarness().addTransactionComponents(component); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java index ff17905..55090ad 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java @@ -19,10 +19,8 @@ package org.apache.hyracks.storage.am.lsm.btree.impls; import java.io.IOException; -import java.util.HashSet; import java.util.LinkedList; import java.util.List; -import java.util.Set; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.exceptions.ErrorCode; @@ -30,10 +28,6 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification; import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.btree.impls.RangePredicate; import org.apache.hyracks.storage.am.common.api.IIndexOperationContext; @@ -44,6 +38,7 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.common.api.ITwoPCIndexBulkLoader; import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; +import org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBloomFilterDiskComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader; @@ -54,7 +49,6 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy; import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker; import org.apache.hyracks.storage.am.lsm.common.api.ITwoPCIndex; @@ -64,22 +58,15 @@ import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationC import org.apache.hyracks.storage.am.lsm.common.impls.ExternalIndexHarness; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor; -import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; import org.apache.hyracks.storage.common.IIndexBulkLoader; import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.ISearchPredicate; -import org.apache.hyracks.storage.common.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeIndex, ITwoPCIndex { - // For creating merge disk components - private final LSMBTreeWithBuddyDiskComponentFactory componentFactory; - - private final LSMBTreeWithBuddyDiskComponentFactory bulkComponentFactory; - private final IBinaryComparatorFactory[] btreeCmpFactories; private final IBinaryComparatorFactory[] buddyBtreeCmpFactories; private final int[] buddyBTreeFields; @@ -96,25 +83,19 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd public ExternalBTreeWithBuddy(IIOManager ioManager, ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory, ITreeIndexFrameFactory buddyBtreeLeafFrameFactory, - IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, - TreeIndexFactory<BTree> bulkLoadBTreeFactory, TreeIndexFactory<BTree> copyBtreeFactory, - TreeIndexFactory<BTree> buddyBtreeFactory, BloomFilterFactory bloomFilterFactory, - double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, - ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, - IBinaryComparatorFactory[] btreeCmpFactories, IBinaryComparatorFactory[] buddyBtreeCmpFactories, - int[] buddyBTreeFields, boolean durable) { + IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory, + ILSMDiskComponentFactory bulkLoadComponentFactory, double bloomFilterFalsePositiveRate, + ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, + ILSMIOOperationCallback ioOpCallback, IBinaryComparatorFactory[] btreeCmpFactories, + IBinaryComparatorFactory[] buddyBtreeCmpFactories, int[] buddyBTreeFields, boolean durable) { super(ioManager, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy, opTracker, - ioScheduler, ioOpCallback, durable); + ioScheduler, ioOpCallback, componentFactory, bulkLoadComponentFactory, durable); this.btreeCmpFactories = btreeCmpFactories; this.buddyBtreeCmpFactories = buddyBtreeCmpFactories; this.buddyBTreeFields = buddyBTreeFields; this.btreeInteriorFrameFactory = btreeInteriorFrameFactory; this.btreeLeafFrameFactory = btreeLeafFrameFactory; this.buddyBtreeLeafFrameFactory = buddyBtreeLeafFrameFactory; - this.componentFactory = - new LSMBTreeWithBuddyDiskComponentFactory(copyBtreeFactory, buddyBtreeFactory, bloomFilterFactory); - this.bulkComponentFactory = - new LSMBTreeWithBuddyDiskComponentFactory(bulkLoadBTreeFactory, buddyBtreeFactory, bloomFilterFactory); this.secondDiskComponents = new LinkedList<>(); } @@ -125,22 +106,16 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd } @Override - protected ILSMDiskComponent loadComponent(LSMComponentFileReferences refs) throws HyracksDataException { - return null; - } - - @Override public void activate() throws HyracksDataException { if (isActive) { throw new HyracksDataException("Failed to activate the index since it is already activated."); } - if (diskComponents.size() == 0 && secondDiskComponents.size() == 0) { //First time activation List<LSMComponentFileReferences> validFileReferences; validFileReferences = fileManager.cleanupAndGetValidFiles(); for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) { - LSMBTreeWithBuddyDiskComponent component; + ILSMDiskComponent component; component = createDiskComponent(componentFactory, lsmComonentFileReference.getInsertIndexFileReference(), lsmComonentFileReference.getDeleteIndexFileReference(), @@ -152,25 +127,13 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd } else { // This index has been opened before or is brand new with no // components. It should also maintain the version pointer - for (ILSMComponent c : diskComponents) { - LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c; - BTree btree = component.getBTree(); - BTree buddyBtree = component.getBuddyBTree(); - BloomFilter bloomFilter = component.getBloomFilter(); - btree.activate(); - buddyBtree.activate(); - bloomFilter.activate(); + for (ILSMDiskComponent c : diskComponents) { + c.activate(false); } - for (ILSMComponent c : secondDiskComponents) { + for (ILSMDiskComponent c : secondDiskComponents) { // Only activate non shared components if (!diskComponents.contains(c)) { - LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c; - BTree btree = component.getBTree(); - BTree buddyBtree = component.getBuddyBTree(); - BloomFilter bloomFilter = component.getBloomFilter(); - btree.activate(); - buddyBtree.activate(); - bloomFilter.activate(); + c.activate(false); } } } @@ -184,12 +147,12 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd } ((ExternalIndexHarness) getLsmHarness()).indexClear(); for (ILSMDiskComponent c : diskComponents) { - clearDiskComponent(c); + c.deactivateAndDestroy(); // Remove from second list to avoid destroying twice secondDiskComponents.remove(c); } for (ILSMDiskComponent c : secondDiskComponents) { - clearDiskComponent(c); + c.deactivateAndDestroy(); } diskComponents.clear(); secondDiskComponents.clear(); @@ -197,28 +160,17 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd } @Override - protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException { - LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c; - component.getBTree().deactivate(); - component.getBuddyBTree().deactivate(); - component.getBloomFilter().deactivate(); - component.getBTree().destroy(); - component.getBloomFilter().destroy(); - component.getBuddyBTree().destroy(); - } - - @Override public void destroy() throws HyracksDataException { if (isActive) { throw new HyracksDataException("Failed to destroy the index since it is activated."); } for (ILSMDiskComponent c : diskComponents) { - destroyDiskComponent(c); + c.destroy(); // Remove from second list to avoid destroying twice secondDiskComponents.remove(c); } for (ILSMDiskComponent c : secondDiskComponents) { - destroyDiskComponent(c); + c.destroy(); } diskComponents.clear(); secondDiskComponents.clear(); @@ -227,14 +179,6 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd } @Override - protected void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException { - LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c; - component.getBTree().destroy(); - component.getBuddyBTree().destroy(); - component.getBloomFilter().destroy(); - } - - @Override public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) throws HyracksDataException { return new LSMTreeIndexAccessor(getLsmHarness(), createOpContext(searchCallback, version), @@ -269,27 +213,6 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd newerList.add(swapIndex, newComponent); } - @Override - public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, - boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter, - boolean cleanupEmptyComponent) throws HyracksDataException { - BloomFilterSpecification bloomFilterSpec = null; - if (numElementsHint > 0) { - int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint); - bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate); - } - if (withFilter && filterFields != null) { - return new LSMBTreeWithBuddyDiskComponentBulkLoader((LSMBTreeWithBuddyDiskComponent) component, - bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, cleanupEmptyComponent, - filterManager, treeFields, filterFields, - MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories())); - } else { - return new LSMBTreeWithBuddyDiskComponentBulkLoader((LSMBTreeWithBuddyDiskComponent) component, - bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, - cleanupEmptyComponent); - } - } - // For initial load @Override public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint) @@ -331,9 +254,9 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd protected LSMComponentFileReferences getMergeTargetFileName(List<ILSMComponent> mergingDiskComponents) throws HyracksDataException { - BTree firstTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(0)).getBTree(); + BTree firstTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(0)).getIndex(); BTree lastTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1)) - .getBTree(); + .getIndex(); FileReference firstFile = firstTree.getFileReference(); FileReference lastFile = lastTree.getFileReference(); LSMComponentFileReferences fileRefs = @@ -385,10 +308,10 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd ILSMIndexOperationContext opCtx = ((LSMBTreeWithBuddySortedCursor) cursor).getOpCtx(); search(opCtx, cursor, btreeSearchPred); - LSMBTreeWithBuddyDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), + ILSMDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBuddyBTreeTarget(), mergeOp.getBloomFilterTarget(), true); - IIndexBulkLoader componentBulkLoader; + ILSMDiskComponentBulkLoader componentBulkLoader; // In case we must keep the deleted-keys BuddyBTrees, then they must be // merged *before* merging the b-trees so that @@ -403,24 +326,23 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd long numElements = 0L; for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) { - numElements += ((LSMBTreeWithBuddyDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter() - .getNumElements(); + numElements += ((AbstractLSMWithBloomFilterDiskComponent) mergeOp.getMergingComponents().get(i)) + .getBloomFilter().getNumElements(); } - componentBulkLoader = - createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false, false); + componentBulkLoader = mergedComponent.createBulkLoader(1.0f, false, numElements, false, false, false); try { while (buddyBtreeCursor.hasNext()) { buddyBtreeCursor.next(); ITupleReference tuple = buddyBtreeCursor.getTuple(); - ((LSMBTreeWithBuddyDiskComponentBulkLoader) componentBulkLoader).delete(tuple); + componentBulkLoader.delete(tuple); } } finally { buddyBtreeCursor.close(); } } else { - componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, 0L, false, false, false); + componentBulkLoader = mergedComponent.createBulkLoader(1.0f, false, 0L, false, false, false); } try { @@ -515,12 +437,12 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd // modify any of the lists to make sure they // are there if the index was opened again for (ILSMDiskComponent c : diskComponents) { - deactivateDiskComponent(c); + c.deactivateAndPurge(); } for (ILSMDiskComponent c : secondDiskComponents) { // Only deactivate non shared components if (!diskComponents.contains(c)) { - deactivateDiskComponent(c); + c.deactivateAndPurge(); } } isActive = false; @@ -564,23 +486,6 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd return btreeCmpFactories; } - private LSMBTreeWithBuddyDiskComponent createDiskComponent(ILSMDiskComponentFactory factory, - FileReference insertFileRef, FileReference deleteFileRef, FileReference bloomFilterFileRef, - boolean createComponent) throws HyracksDataException { - // Create new instance. - LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) factory - .createComponent(new LSMComponentFileReferences(insertFileRef, deleteFileRef, bloomFilterFileRef)); - if (createComponent) { - component.getBTree().create(); - component.getBuddyBTree().create(); - component.getBloomFilter().create(); - } - component.getBTree().activate(); - component.getBuddyBTree().activate(); - component.getBloomFilter().activate(); - return component; - } - // even though the index doesn't support record level modification, the // accessor will try to do it // we could throw the exception here but we don't. it will eventually be @@ -590,7 +495,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd // modifications public class LSMTwoPCBTreeWithBuddyBulkLoader implements IIndexBulkLoader, ITwoPCIndexBulkLoader { private final ILSMDiskComponent component; - private final IIndexBulkLoader componentBulkLoader; + private final ILSMDiskComponentBulkLoader componentBulkLoader; private final boolean isTransaction; public LSMTwoPCBTreeWithBuddyBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint, @@ -604,7 +509,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd } componentBulkLoader = - createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true, false); + component.createBulkLoader(fillFactor, verifyInput, numElementsHint, false, true, false); } @Override @@ -620,12 +525,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd // Since this is a transaction component, validate and // deactivate. it could later be added or deleted component.markAsValid(durable); - BTree btree = ((LSMBTreeWithBuddyDiskComponent) component).getBTree(); - BTree buddyBtree = ((LSMBTreeWithBuddyDiskComponent) component).getBuddyBTree(); - BloomFilter bloomFilter = ((LSMBTreeWithBuddyDiskComponent) component).getBloomFilter(); - btree.deactivate(); - buddyBtree.deactivate(); - bloomFilter.deactivate(); + component.deactivate(); } else { getLsmHarness().addBulkLoadedComponent(component); } @@ -634,7 +534,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd @Override public void delete(ITupleReference tuple) throws HyracksDataException { - ((LSMBTreeWithBuddyDiskComponentBulkLoader) componentBulkLoader).delete(tuple); + componentBulkLoader.delete(tuple); } @Override @@ -654,20 +554,13 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd } catch (IOException e) { throw HyracksDataException.create(e); } - return createDiskComponent(bulkComponentFactory, componentFileRefs.getInsertIndexFileReference(), + return createDiskComponent(bulkLoadComponentFactory, componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true); } } @Override - public ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException { - LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference(); - return createDiskComponent(bulkComponentFactory, componentFileRefs.getInsertIndexFileReference(), - componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true); - } - - @Override public ILSMIndexAccessor createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion) throws HyracksDataException { return new LSMTreeIndexAccessor(getLsmHarness(), createOpContext(searchCallback, targetIndexVersion), @@ -709,7 +602,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd @Override public void commitTransaction() throws HyracksDataException { LSMComponentFileReferences componentFileRefrences = fileManager.getTransactionFileReferenceForCommit(); - LSMBTreeWithBuddyDiskComponent component = null; + ILSMDiskComponent component = null; if (componentFileRefrences != null) { component = createDiskComponent(componentFactory, componentFileRefrences.getInsertIndexFileReference(), componentFileRefrences.getDeleteIndexFileReference(), @@ -739,60 +632,6 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd } @Override - public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) { - Set<String> files = new HashSet<>(); - LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) lsmComponent; - files.add(component.getBTree().getFileReference().getFile().getAbsolutePath()); - files.add(component.getBuddyBTree().getFileReference().getFile().getAbsolutePath()); - files.add(component.getBloomFilter().getFileReference().getFile().getAbsolutePath()); - return files; - } - - @Override - protected void deactivateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { - //do nothing since external index never use memory components - } - - @Override - protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException { - LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c; - BTree btree = component.getBTree(); - BTree buddyBtree = component.getBuddyBTree(); - BloomFilter bloomFilter = component.getBloomFilter(); - btree.deactivate(); - buddyBtree.deactivate(); - bloomFilter.deactivate(); - btree.purge(); - buddyBtree.purge(); - bloomFilter.purge(); - } - - @Override - protected void clearMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { - //do nothing since external index never use memory components - } - - @Override - protected long getMemoryComponentSize(ILSMMemoryComponent c) { - return 0; - } - - @Override - protected void validateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { - throw new UnsupportedOperationException("Validation not implemented for LSM B-Trees with Buddy B-Tree."); - } - - @Override - protected void validateDiskComponent(ILSMDiskComponent c) throws HyracksDataException { - throw new UnsupportedOperationException("Validation not implemented for LSM B-Trees with Buddy B-Tree."); - } - - @Override - protected void allocateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { - //do nothing since external index never use memory components - } - - @Override protected LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent, ILSMDiskComponent lastComponent) throws HyracksDataException { return null; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java index c7d45e1..f6942b0 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java @@ -20,9 +20,7 @@ package org.apache.hyracks.storage.am.lsm.btree.impls; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.exceptions.ErrorCode; @@ -31,9 +29,6 @@ import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.data.std.primitive.IntegerPointable; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification; import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor; import org.apache.hyracks.storage.am.btree.impls.RangePredicate; @@ -44,18 +39,19 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback; import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleReference; +import org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBloomFilterDiskComponent; import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy; import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker; import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; @@ -66,9 +62,7 @@ import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager; import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor; import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor.ICursorFactory; -import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; import org.apache.hyracks.storage.common.IIndexAccessor; -import org.apache.hyracks.storage.common.IIndexBulkLoader; import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.ISearchOperationCallback; @@ -80,12 +74,6 @@ import org.apache.hyracks.util.trace.ITracer; public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { private static final ICursorFactory cursorFactory = opCtx -> new LSMBTreeSearchCursor(opCtx); - // For creating BTree's used in flush and merge. - protected final LSMBTreeDiskComponentFactory componentFactory; - // For creating BTree's used in bulk load. Different from diskBTreeFactory - // because it should have a different tuple writer in it's leaf frames. - protected final LSMBTreeDiskComponentFactory bulkLoadComponentFactory; - // Common for in-memory and on-disk components. protected final ITreeIndexFrameFactory insertLeafFrameFactory; protected final ITreeIndexFrameFactory deleteLeafFrameFactory; @@ -99,16 +87,16 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { public LSMBTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory, - ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager, - TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory, - BloomFilterFactory bloomFilterFactory, IComponentFilterHelper filterHelper, + ITreeIndexFrameFactory deleteLeafFrameFactory, IBufferCache diskBufferCache, + ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory, + ILSMDiskComponentFactory bulkLoadComponentFactory, IComponentFilterHelper filterHelper, ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, double bloomFilterFalsePositiveRate, int fieldCount, IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, int[] btreeFields, int[] filterFields, boolean durable, boolean updateAware, ITracer tracer) throws HyracksDataException { - super(ioManager, virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, - bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory, + super(ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy, + opTracker, ioScheduler, ioOpCallback, componentFactory, bulkLoadComponentFactory, filterFrameFactory, filterManager, filterFields, durable, filterHelper, btreeFields, tracer); this.insertLeafFrameFactory = insertLeafFrameFactory; this.deleteLeafFrameFactory = deleteLeafFrameFactory; @@ -125,35 +113,25 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { memoryComponents.add(mutableComponent); ++i; } - componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, filterHelper); - bulkLoadComponentFactory = - new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory, filterHelper); this.needKeyDupCheck = needKeyDupCheck; this.hasBloomFilter = needKeyDupCheck; } // Without memory components public LSMBTree(IIOManager ioManager, ITreeIndexFrameFactory insertLeafFrameFactory, - ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager, - TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory, - BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate, - IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, - ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, - boolean durable) { - super(ioManager, diskBTreeFactory.getBufferCache(), fileManager, bloomFilterFalsePositiveRate, mergePolicy, - opTracker, ioScheduler, ioOpCallback, durable); + ITreeIndexFrameFactory deleteLeafFrameFactory, IBufferCache bufferCache, ILSMIndexFileManager fileManager, + ILSMDiskComponentFactory componentFactory, ILSMDiskComponentFactory bulkLoadComponentFactory, + double bloomFilterFalsePositiveRate, IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, + ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, + boolean needKeyDupCheck, boolean durable) { + super(ioManager, bufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, + ioOpCallback, componentFactory, bulkLoadComponentFactory, durable); this.insertLeafFrameFactory = insertLeafFrameFactory; this.deleteLeafFrameFactory = deleteLeafFrameFactory; this.cmpFactories = cmpFactories; this.needKeyDupCheck = needKeyDupCheck; this.hasBloomFilter = true; this.updateAware = false; - componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, null); - bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory, null); - } - - public boolean hasBloomFilter() { - return hasBloomFilter; } @Override @@ -167,35 +145,6 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { } @Override - protected ILSMDiskComponent loadComponent(LSMComponentFileReferences lsmComonentFileReferences) - throws HyracksDataException { - return createDiskComponent(componentFactory, lsmComonentFileReferences.getInsertIndexFileReference(), - lsmComonentFileReferences.getBloomFilterFileReference(), false); - } - - @Override - protected void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException { - LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; - component.getBTree().destroy(); - if (hasBloomFilter) { - component.getBloomFilter().destroy(); - } - } - - @Override - protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException { - LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; - if (hasBloomFilter) { - component.getBloomFilter().deactivate(); - } - component.getBTree().deactivate(); - if (hasBloomFilter) { - component.getBloomFilter().destroy(); - } - component.getBTree().destroy(); - } - - @Override public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException { LSMBTreeOpContext ctx = (LSMBTreeOpContext) ictx; ITupleReference indexTuple; @@ -297,7 +246,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { public ILSMDiskComponent doFlush(ILSMIOOperation operation) throws HyracksDataException { LSMBTreeFlushOperation flushOp = (LSMBTreeFlushOperation) operation; LSMBTreeMemoryComponent flushingComponent = (LSMBTreeMemoryComponent) flushOp.getFlushingComponent(); - IIndexAccessor accessor = flushingComponent.getBTree().createAccessor(NoOpOperationCallback.INSTANCE, + IIndexAccessor accessor = flushingComponent.getIndex().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null); @@ -317,11 +266,11 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { } } - LSMBTreeDiskComponent component = - createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getBloomFilterTarget(), true); + ILSMDiskComponent component = + createDiskComponent(componentFactory, flushOp.getTarget(), null, flushOp.getBloomFilterTarget(), true); ILSMDiskComponentBulkLoader componentBulkLoader = - createComponentBulkLoader(component, 1.0f, false, numElements, false, false, false); + component.createBulkLoader(1.0f, false, numElements, false, false, false); IIndexCursor scanCursor = accessor.createSearchCursor(false); accessor.search(scanCursor, nullPred); @@ -344,7 +293,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple()); filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple()); getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples); - getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getBTree()); + getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder()); } // Write metadata from memory component to disk // Q. what about the merge operation? how do we resolve conflicts @@ -370,14 +319,15 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { if (hasBloomFilter) { //count elements in btree for creating Bloomfilter for (int i = 0; i < mergedComponents.size(); ++i) { - numElements += ((LSMBTreeDiskComponent) mergedComponents.get(i)).getBloomFilter().getNumElements(); + numElements += ((AbstractLSMWithBloomFilterDiskComponent) mergedComponents.get(i)).getBloomFilter() + .getNumElements(); } } - LSMBTreeDiskComponent mergedComponent = - createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBloomFilterTarget(), true); + ILSMDiskComponent mergedComponent = + createDiskComponent(componentFactory, mergeOp.getTarget(), null, mergeOp.getBloomFilterTarget(), true); ILSMDiskComponentBulkLoader componentBulkLoader = - createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false, false); + mergedComponent.createBulkLoader(1.0f, false, numElements, false, false, false); try { while (cursor.hasNext()) { cursor.next(); @@ -394,69 +344,13 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple()); } getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples); - getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getBTree()); + getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(), + mergedComponent.getMetadataHolder()); } componentBulkLoader.end(); return mergedComponent; } - protected LSMBTreeDiskComponent createDiskComponent(LSMBTreeDiskComponentFactory factory, - FileReference btreeFileRef, FileReference bloomFilterFileRef, boolean createComponent) - throws HyracksDataException { - // Create new BTree instance. - LSMBTreeDiskComponent component = - factory.createComponent(new LSMComponentFileReferences(btreeFileRef, null, bloomFilterFileRef)); - // BTree will be closed during cleanup of merge(). - if (createComponent) { - component.getBTree().create(); - } - component.getBTree().activate(); - if (hasBloomFilter) { - if (createComponent) { - component.getBloomFilter().create(); - } - component.getBloomFilter().activate(); - } - if (component.getLSMComponentFilter() != null && !createComponent) { - getFilterManager().readFilter(component.getLSMComponentFilter(), component.getBTree()); - } - return component; - } - - @Override - public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, - boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter, - boolean cleanupEmptyComponent) throws HyracksDataException { - BloomFilterSpecification bloomFilterSpec = null; - if (hasBloomFilter) { - int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint); - bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate); - } - - if (withFilter && filterFields != null) { - return new LSMBTreeDiskComponentBulkLoader((LSMBTreeDiskComponent) component, bloomFilterSpec, fillFactor, - verifyInput, numElementsHint, checkIfEmptyIndex, cleanupEmptyComponent, filterManager, treeFields, - filterFields, MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories())); - } else { - return new LSMBTreeDiskComponentBulkLoader((LSMBTreeDiskComponent) component, bloomFilterSpec, fillFactor, - verifyInput, numElementsHint, checkIfEmptyIndex, cleanupEmptyComponent); - } - - } - - @Override - public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint) - throws HyracksDataException { - return new LSMBTreeBulkLoader(this, fillLevel, verifyInput, numElementsHint); - } - - @Override - public ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException { - LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference(); - return createDiskComponent(bulkLoadComponentFactory, componentFileRefs.getInsertIndexFileReference(), - componentFileRefs.getBloomFilterFileReference(), true); - } - @Override protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx, LSMComponentFileReferences componentFileRefs, ILSMIOOperationCallback callback) { @@ -468,7 +362,9 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { @Override public LSMBTreeOpContext createOpContext(IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) { - int numBloomFilterKeyFields = hasBloomFilter ? componentFactory.getBloomFilterKeyFields().length : 0; + int numBloomFilterKeyFields = hasBloomFilter + ? ((LSMBTreeWithBloomFilterDiskComponentFactory) componentFactory).getBloomFilterKeyFields().length + : 0; return new LSMBTreeOpContext(memoryComponents, insertLeafFrameFactory, deleteLeafFrameFactory, modificationCallback, searchCallback, numBloomFilterKeyFields, getTreeFields(), getFilterFields(), getLsmHarness(), getFilterCmpFactories()); @@ -488,113 +384,49 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { public ITreeIndexFrameFactory getInteriorFrameFactory() { LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get()); - return mutableComponent.getBTree().getInteriorFrameFactory(); + return mutableComponent.getIndex().getInteriorFrameFactory(); } @Override public int getFieldCount() { LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get()); - return mutableComponent.getBTree().getFieldCount(); + return mutableComponent.getIndex().getFieldCount(); } @Override public int getFileId() { LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get()); - return mutableComponent.getBTree().getFileId(); + return mutableComponent.getIndex().getFileId(); } @Override public IPageManager getPageManager() { LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get()); - return mutableComponent.getBTree().getPageManager(); + return mutableComponent.getIndex().getPageManager(); } @Override public ITreeIndexFrameFactory getLeafFrameFactory() { LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get()); - return mutableComponent.getBTree().getLeafFrameFactory(); + return mutableComponent.getIndex().getLeafFrameFactory(); } @Override public int getRootPageId() { LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get()); - return mutableComponent.getBTree().getRootPageId(); - } - - @Override - protected long getMemoryComponentSize(ILSMMemoryComponent c) { - LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c; - IBufferCache virtualBufferCache = mutableComponent.getBTree().getBufferCache(); - return virtualBufferCache.getNumPages() * (long) virtualBufferCache.getPageSize(); - } - - @Override - public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) { - Set<String> files = new HashSet<>(); - LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) lsmComponent; - files.add(component.getBTree().getFileReference().getFile().getAbsolutePath()); - if (hasBloomFilter) { - files.add(component.getBloomFilter().getFileReference().getFile().getAbsolutePath()); - } - return files; - } - - @Override - protected void clearMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { - LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c; - mutableComponent.getBTree().clear(); - mutableComponent.reset(); - } - - @Override - protected void validateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { - LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c; - mutableComponent.getBTree().validate(); - } - - @Override - protected void validateDiskComponent(ILSMDiskComponent c) throws HyracksDataException { - BTree btree = ((LSMBTreeDiskComponent) c).getBTree(); - btree.validate(); - } - - @Override - protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException { - LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; - component.getBTree().deactivate(); - component.getBTree().purge(); - if (hasBloomFilter) { - component.getBloomFilter().deactivate(); - component.getBloomFilter().purge(); - } - } - - @Override - protected void deactivateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { - LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c; - mutableComponent.getBTree().deactivate(); - mutableComponent.getBTree().destroy(); - ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).close(); - } - - @Override - protected void allocateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { - LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c; - ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).open(); - mutableComponent.getBTree().create(); - mutableComponent.getBTree().activate(); + return mutableComponent.getIndex().getRootPageId(); } @Override protected LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent, ILSMDiskComponent lastComponent) throws HyracksDataException { - BTree firstBTree = ((LSMBTreeDiskComponent) firstComponent).getBTree(); - BTree lastBTree = ((LSMBTreeDiskComponent) lastComponent).getBTree(); + BTree firstBTree = (BTree) firstComponent.getIndex(); + BTree lastBTree = (BTree) lastComponent.getIndex(); FileReference firstFile = firstBTree.getFileReference(); FileReference lastFile = lastBTree.getFileReference(); return fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
