Repository: asterixdb Updated Branches: refs/heads/master 536aa3f2b -> 77f895257
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java index 1e15455..cfb05e7 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java @@ -20,18 +20,16 @@ package org.apache.hyracks.storage.am.lsm.rtree.impls; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; +import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor; import org.apache.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor; import org.apache.hyracks.storage.am.btree.impls.RangePredicate; -import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor; import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback; @@ -50,7 +48,6 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy; import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker; import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; -import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager; @@ -58,30 +55,25 @@ import org.apache.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor; import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor; import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor.ICursorFactory; import org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation; -import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; import org.apache.hyracks.storage.am.rtree.frames.RTreeFrameFactory; import org.apache.hyracks.storage.am.rtree.impls.RTree; +import org.apache.hyracks.storage.am.rtree.impls.RTree.RTreeAccessor; import org.apache.hyracks.storage.am.rtree.impls.RTreeSearchCursor; import org.apache.hyracks.storage.am.rtree.impls.SearchPredicate; -import org.apache.hyracks.storage.common.IIndexBulkLoader; import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.ISearchPredicate; -import org.apache.hyracks.storage.common.MultiComparator; +import org.apache.hyracks.storage.common.buffercache.IBufferCache; public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree { private static final ICursorFactory cursorFactory = opCtx -> new LSMRTreeWithAntiMatterTuplesSearchCursor(opCtx); - // On-disk components. - // For creating RTree's used in bulk load. Different from diskRTreeFactory - // because it should have a different tuple writer in it's leaf frames. - private final ILSMDiskComponentFactory bulkLoaComponentFactory; public LSMRTreeWithAntiMatterTuples(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, RTreeFrameFactory rtreeInteriorFrameFactory, RTreeFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory, - ILSMIndexFileManager fileManager, TreeIndexFactory<RTree> diskRTreeFactory, - TreeIndexFactory<RTree> bulkLoadRTreeFactory, IComponentFilterHelper filterHelper, + IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory, + ILSMDiskComponentFactory bulkLoadComponentFactory, IComponentFilterHelper filterHelper, ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, @@ -89,37 +81,10 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree { ILSMIOOperationCallback ioOpCallback, int[] rtreeFields, int[] filterFields, boolean durable, boolean isPointMBR) throws HyracksDataException { super(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, - btreeInteriorFrameFactory, btreeLeafFrameFactory, fileManager, - new LSMRTreeWithAntiMatterTuplesDiskComponentFactory(diskRTreeFactory, filterHelper), fieldCount, - rtreeCmpFactories, btreeComparatorFactories, linearizer, comparatorFields, linearizerArray, 0, - mergePolicy, opTracker, ioScheduler, ioOpCallback, filterHelper, filterFrameFactory, filterManager, - rtreeFields, filterFields, durable, isPointMBR, diskRTreeFactory.getBufferCache()); - bulkLoaComponentFactory = - new LSMRTreeWithAntiMatterTuplesDiskComponentFactory(bulkLoadRTreeFactory, filterHelper); - } - - @Override - protected ILSMDiskComponent loadComponent(LSMComponentFileReferences refs) throws HyracksDataException { - return createDiskComponent(componentFactory, refs.getInsertIndexFileReference(), null, null, false); - } - - @Override - protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException { - RTree rtree = ((LSMRTreeDiskComponent) c).getRTree(); - rtree.deactivate(); - rtree.purge(); - } - - @Override - protected void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException { - ((LSMRTreeDiskComponent) c).getRTree().destroy(); - } - - @Override - protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException { - RTree rtree = ((LSMRTreeDiskComponent) c).getRTree(); - rtree.deactivate(); - rtree.destroy(); + btreeInteriorFrameFactory, btreeLeafFrameFactory, diskBufferCache, fileManager, componentFactory, + bulkLoadComponentFactory, fieldCount, rtreeCmpFactories, btreeComparatorFactories, linearizer, + comparatorFields, linearizerArray, 0, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterHelper, + filterFrameFactory, filterManager, rtreeFields, filterFields, durable, isPointMBR); } @Override @@ -129,20 +94,20 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree { // read the file names when we open the tree. // The RTree should be renamed before the BTree. LSMRTreeMemoryComponent flushingComponent = (LSMRTreeMemoryComponent) flushOp.getFlushingComponent(); - ITreeIndexAccessor memRTreeAccessor = flushingComponent.getRTree() - .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); - RTreeSearchCursor rtreeScanCursor = (RTreeSearchCursor) memRTreeAccessor.createSearchCursor(false); + RTreeAccessor memRTreeAccessor = flushingComponent.getIndex().createAccessor(NoOpOperationCallback.INSTANCE, + NoOpOperationCallback.INSTANCE); + RTreeSearchCursor rtreeScanCursor = memRTreeAccessor.createSearchCursor(false); SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null); memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate); - LSMRTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), null, null, true); + ILSMDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), null, null, true); ILSMDiskComponentBulkLoader componentBulkLoader = - createComponentBulkLoader(component, 1.0f, false, 0L, false, false, false); + component.createBulkLoader(1.0f, false, 0L, false, false, false); // Since the LSM-RTree is used as a secondary assumption, the // primary key will be the last comparator in the BTree comparators - TreeTupleSorter rTreeTupleSorter = new TreeTupleSorter(flushingComponent.getRTree().getFileId(), + TreeTupleSorter rTreeTupleSorter = new TreeTupleSorter(flushingComponent.getIndex().getFileId(), linearizerArray, rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(), - flushingComponent.getRTree().getBufferCache(), comparatorFields); + flushingComponent.getIndex().getBufferCache(), comparatorFields); boolean isEmpty = true; try { @@ -159,14 +124,14 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree { } // scan the memory BTree - ITreeIndexAccessor memBTreeAccessor = flushingComponent.getBTree() + BTreeAccessor memBTreeAccessor = flushingComponent.getBuddyIndex() .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); - BTreeRangeSearchCursor btreeScanCursor = (BTreeRangeSearchCursor) memBTreeAccessor.createSearchCursor(false); + BTreeRangeSearchCursor btreeScanCursor = memBTreeAccessor.createSearchCursor(false); RangePredicate btreeNullPredicate = new RangePredicate(null, null, true, true, null, null); memBTreeAccessor.search(btreeScanCursor, btreeNullPredicate); - TreeTupleSorter bTreeTupleSorter = new TreeTupleSorter(flushingComponent.getBTree().getFileId(), + TreeTupleSorter bTreeTupleSorter = new TreeTupleSorter(flushingComponent.getBuddyIndex().getFileId(), linearizerArray, btreeLeafFrameFactory.createFrame(), btreeLeafFrameFactory.createFrame(), - flushingComponent.getBTree().getBufferCache(), comparatorFields); + flushingComponent.getBuddyIndex().getBufferCache(), comparatorFields); isEmpty = true; try { @@ -202,7 +167,7 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree { filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple()); filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple()); getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples); - getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getRTree()); + getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder()); } flushingComponent.getMetadata().copy(component.getMetadata()); @@ -219,10 +184,10 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree { search(opCtx, cursor, rtreeSearchPred); // Bulk load the tuples from all on-disk RTrees into the new RTree. - LSMRTreeDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(), null, null, true); + ILSMDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(), null, null, true); ILSMDiskComponentBulkLoader componentBulkLoader = - createComponentBulkLoader(component, 1.0f, false, 0L, false, false, false); + component.createBulkLoader(1.0f, false, 0L, false, false, false); try { while (cursor.hasNext()) { cursor.next(); @@ -239,7 +204,7 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree { filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple()); } getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples); - getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getRTree()); + getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder()); } componentBulkLoader.end(); @@ -255,78 +220,6 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree { } @Override - public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, - boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter, - boolean cleanupEmptyComponent) throws HyracksDataException { - if (withFilter && filterFields != null) { - return new LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader((LSMRTreeDiskComponent) component, null, - fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, cleanupEmptyComponent, filterManager, - treeFields, filterFields, - MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories())); - } else { - return new LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader((LSMRTreeDiskComponent) component, null, - fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, cleanupEmptyComponent); - } - } - - @Override - public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint) - throws HyracksDataException { - return new LSMRTreeWithAntiMatterTuplesBulkLoader(fillLevel, verifyInput, numElementsHint); - } - - public class LSMRTreeWithAntiMatterTuplesBulkLoader implements IIndexBulkLoader { - private final ILSMDiskComponent component; - private final IIndexBulkLoader componentBulkLoader; - - public LSMRTreeWithAntiMatterTuplesBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint) - throws HyracksDataException { - component = createBulkLoadTarget(); - - componentBulkLoader = - createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true, true); - } - - @Override - public void add(ITupleReference tuple) throws HyracksDataException { - componentBulkLoader.add(tuple); - } - - @Override - public void end() throws HyracksDataException { - - componentBulkLoader.end(); - if (component.getComponentSize() > 0) { - ioOpCallback.afterOperation(LSMOperationType.FLUSH, null, component); - lsmHarness.addBulkLoadedComponent(component); - } - } - - @Override - public void abort() throws HyracksDataException { - if (componentBulkLoader != null) { - componentBulkLoader.abort(); - } - } - - } - - @Override - public ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException { - LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference(); - return createDiskComponent(bulkLoaComponentFactory, relFlushFileRefs.getInsertIndexFileReference(), null, null, - true); - } - - @Override - public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) { - Set<String> files = new HashSet<>(); - RTree rtree = ((LSMRTreeDiskComponent) lsmComponent).getRTree(); - files.add(rtree.getFileReference().getFile().getAbsolutePath()); - return files; - } - - @Override protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx, LSMComponentFileReferences componentFileRefs, ILSMIOOperationCallback callback) throws HyracksDataException { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java deleted file mode 100644 index 22046aa..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hyracks.storage.am.lsm.rtree.impls; - -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification; -import org.apache.hyracks.storage.am.common.api.ITreeIndex; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterManager; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; -import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponentBulkLoader; -import org.apache.hyracks.storage.common.MultiComparator; - -public class LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader extends AbstractLSMDiskComponentBulkLoader { - - //with filter - public LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader(LSMRTreeDiskComponent component, - BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint, - boolean checkIfEmptyIndex, boolean cleanupEmptyComponent, ILSMComponentFilterManager filterManager, - int[] indexFields, int[] filterFields, MultiComparator filterCmp) throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, - cleanupEmptyComponent, filterManager, indexFields, filterFields, filterCmp); - } - - //without filter - public LSMRTreeWithAntiMatterTuplesDiskComponentBulkLoader(LSMRTreeDiskComponent component, - BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint, - boolean checkIfEmptyIndex, boolean cleanupEmptyComponent) throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, - cleanupEmptyComponent, null, null, null, null); - } - - @Override - protected BloomFilter getBloomFilter(ILSMDiskComponent component) { - return ((LSMRTreeDiskComponent) component).getBloomFilter(); - } - - @Override - protected ITreeIndex getIndex(ILSMDiskComponent component) { - return ((LSMRTreeDiskComponent) component).getRTree(); - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentFactory.java index a6216fc..6eae415 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentFactory.java @@ -22,6 +22,7 @@ package org.apache.hyracks.storage.am.lsm.rtree.impls; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; import org.apache.hyracks.storage.am.rtree.impls.RTree; @@ -37,8 +38,10 @@ public class LSMRTreeWithAntiMatterTuplesDiskComponentFactory implements ILSMDis } @Override - public LSMRTreeDiskComponent createComponent(LSMComponentFileReferences cfr) throws HyracksDataException { - return new LSMRTreeDiskComponent(rtreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()), null, - null, filterHelper == null ? null : filterHelper.createFilter()); + public LSMRTreeWithAntimatterDiskComponent createComponent(AbstractLSMIndex lsmIndex, + LSMComponentFileReferences cfr) throws HyracksDataException { + return new LSMRTreeWithAntimatterDiskComponent(lsmIndex, + rtreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()), + filterHelper == null ? null : filterHelper.createFilter()); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java index 81917ee..4abab53 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java @@ -101,8 +101,8 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs btreeAccessors = new ITreeIndexAccessor[numMutableComponents]; for (int i = 0; i < numMutableComponents; i++) { ILSMComponent component = operationalComponents.get(i); - RTree rtree = ((LSMRTreeMemoryComponent) component).getRTree(); - BTree btree = ((LSMRTreeMemoryComponent) component).getBTree(); + RTree rtree = ((LSMRTreeMemoryComponent) component).getIndex(); + BTree btree = ((LSMRTreeMemoryComponent) component).getBuddyIndex(); mutableRTreeCursors[i] = new RTreeSearchCursor( (IRTreeInteriorFrame) lsmInitialState.getRTreeInteriorFrameFactory().createFrame(), (IRTreeLeafFrame) lsmInitialState.getRTreeLeafFrameFactory().createFrame()); @@ -121,7 +121,7 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs rangeCursors[j] = new RTreeSearchCursor( (IRTreeInteriorFrame) lsmInitialState.getRTreeInteriorFrameFactory().createFrame(), (IRTreeLeafFrame) lsmInitialState.getRTreeLeafFrameFactory().createFrame()); - RTree rtree = ((LSMRTreeDiskComponent) component).getRTree(); + RTree rtree = ((LSMRTreeWithAntimatterDiskComponent) component).getIndex(); immutableRTreeAccessors[j] = rtree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); immutableRTreeAccessors[j].search(rangeCursors[j], searchPred); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntimatterDiskComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntimatterDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntimatterDiskComponent.java new file mode 100644 index 0000000..eed01a8 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntimatterDiskComponent.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.storage.am.lsm.rtree.impls; + +import java.util.Set; + +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex; +import org.apache.hyracks.storage.am.rtree.impls.RTree; + +public class LSMRTreeWithAntimatterDiskComponent extends AbstractLSMDiskComponent { + private final RTree rtree; + + public LSMRTreeWithAntimatterDiskComponent(AbstractLSMIndex lsmIndex, RTree rtree, ILSMComponentFilter filter) { + super(lsmIndex, LSMRTreeDiskComponent.getMetadataPageManager(rtree), filter); + this.rtree = rtree; + } + + @Override + public RTree getIndex() { + return rtree; + } + + @Override + public RTree getMetadataHolder() { + return rtree; + } + + @Override + public long getComponentSize() { + return LSMRTreeDiskComponent.getComponentSize(rtree); + } + + @Override + public int getFileReferenceCount() { + return LSMRTreeDiskComponent.getFileReferenceCount(rtree); + } + + @Override + public Set<String> getLSMComponentPhysicalFiles() { + return LSMRTreeDiskComponent.getFiles(rtree); + } + + @Override + public String toString() { + return getClass().getSimpleName() + ":" + rtree.getFileReference().getRelativePath(); + } + + @Override + public void validate() throws HyracksDataException { + throw new UnsupportedOperationException("Validation not implemented for LSM R-Trees."); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java index 5e3a198..d1a3195 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java @@ -39,6 +39,7 @@ import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory; import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager; @@ -52,8 +53,10 @@ import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager; import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; import org.apache.hyracks.storage.am.lsm.rtree.impls.ExternalRTree; import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTree; +import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeFileManager; import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeWithAntiMatterTuples; +import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeWithAntiMatterTuplesDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeWithAntiMatterTuplesFileManager; import org.apache.hyracks.storage.am.lsm.rtree.impls.RTreeFactory; import org.apache.hyracks.storage.am.lsm.rtree.tuples.LSMRTreeCopyTupleWriterFactory; @@ -129,12 +132,15 @@ public class LSMRTreeUtils { } ILSMIndexFileManager fileNameManager = new LSMRTreeFileManager(ioManager, file, diskRTreeFactory, diskBTreeFactory); + ILSMDiskComponentFactory componentFactory = + new LSMRTreeDiskComponentFactory(diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, filterHelper); + return new LSMRTree(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, - btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, diskBTreeFactory, - bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, - typeTraits.length, rtreeCmpFactories, btreeCmpFactories, linearizeCmpFactory, comparatorFields, - linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, rtreeFields, buddyBTreeFields, - filterFields, durable, isPointMBR); + btreeInteriorFrameFactory, btreeLeafFrameFactory, diskBufferCache, fileNameManager, componentFactory, + filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, typeTraits.length, + rtreeCmpFactories, btreeCmpFactories, linearizeCmpFactory, comparatorFields, linearizerArray, + mergePolicy, opTracker, ioScheduler, ioOpCallback, rtreeFields, buddyBTreeFields, filterFields, durable, + isPointMBR); } public static LSMRTreeWithAntiMatterTuples createLSMTreeWithAntiMatterTuples(IIOManager ioManager, @@ -216,12 +222,18 @@ public class LSMRTreeUtils { } ILSMIndexFileManager fileNameManager = new LSMRTreeWithAntiMatterTuplesFileManager(ioManager, file, diskRTreeFactory); + + ILSMDiskComponentFactory componentFactory = + new LSMRTreeWithAntiMatterTuplesDiskComponentFactory(diskRTreeFactory, filterHelper); + ILSMDiskComponentFactory bulkLoadComponentFactory = + new LSMRTreeWithAntiMatterTuplesDiskComponentFactory(bulkLoadRTreeFactory, filterHelper); + return new LSMRTreeWithAntiMatterTuples(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, - rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager, - diskRTreeFactory, bulkLoadRTreeFactory, filterHelper, filterFrameFactory, filterManager, - typeTraits.length, rtreeCmpFactories, btreeComparatorFactories, linearizerCmpFactory, comparatorFields, - linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, rtreeFields, filterFields, durable, - isPointMBR); + rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, diskBufferCache, + fileNameManager, componentFactory, bulkLoadComponentFactory, filterHelper, filterFrameFactory, + filterManager, typeTraits.length, rtreeCmpFactories, btreeComparatorFactories, linearizerCmpFactory, + comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, rtreeFields, + filterFields, durable, isPointMBR); } public static ExternalRTree createExternalRTree(IIOManager ioManager, FileReference file, @@ -270,11 +282,14 @@ public class LSMRTreeUtils { ILSMIndexFileManager fileNameManager = new LSMRTreeFileManager(ioManager, file, diskRTreeFactory, diskBTreeFactory); + ILSMDiskComponentFactory componentFactory = + new LSMRTreeDiskComponentFactory(diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, null); + return new ExternalRTree(ioManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, - btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, - bloomFilterFalsePositiveRate, typeTraits.length, rtreeCmpFactories, btreeCmpFactories, - linearizeCmpFactory, comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler, - ioOpCallback, buddyBTreeFields, durable, isPointMBR); + btreeLeafFrameFactory, diskBufferCache, fileNameManager, componentFactory, bloomFilterFalsePositiveRate, + typeTraits.length, rtreeCmpFactories, btreeCmpFactories, linearizeCmpFactory, comparatorFields, + linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, buddyBTreeFields, durable, + isPointMBR); } public static ILinearizeComparatorFactory proposeBestLinearizer(ITypeTraits[] typeTraits, int numKeyFields) http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java index 9a39630..ee7c827 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java @@ -756,12 +756,12 @@ public class RTree extends AbstractTreeIndex { } @Override - public ITreeIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, + public RTreeAccessor createAccessor(IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) { return new RTreeAccessor(this, modificationCallback, searchCallback); } - public ITreeIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, + public RTreeAccessor createAccessor(IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback, int[] nonIndexFields) { return new RTreeAccessor(this, modificationCallback, searchCallback, nonIndexFields); } @@ -807,7 +807,7 @@ public class RTree extends AbstractTreeIndex { } @Override - public ITreeIndexCursor createSearchCursor(boolean exclusive) { + public RTreeSearchCursor createSearchCursor(boolean exclusive) { return new RTreeSearchCursor((IRTreeInteriorFrame) interiorFrameFactory.createFrame(), (IRTreeLeafFrame) leafFrameFactory.createFrame()); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndex.java index 255e0d0..473cfcb 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndex.java @@ -86,6 +86,15 @@ public interface IIndex { void destroy() throws HyracksDataException; /** + * Purge the index files out of the buffer cache. + * Can only be called if the caller is absolutely sure the files don't contain dirty pages + * + * @throws HyracksDataException + * if the index is active + */ + void purge() throws HyracksDataException; + + /** * Creates an {@link IIndexAccessor} for performing operations on this index. * An IIndexAccessor is not thread safe, but different IIndexAccessors can concurrently operate * on the same {@link IIndex}. @@ -101,6 +110,8 @@ public interface IIndex { ISearchOperationCallback searchCallback) throws HyracksDataException; /** + * TODO: Get rid of this method + * * Strictly a test method * * Ensures that all pages (and tuples) of the index are logically consistent. http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/IndexTestContext.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/IndexTestContext.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/IndexTestContext.java index 4213a34..3e1c922 100644 --- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/IndexTestContext.java +++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/IndexTestContext.java @@ -40,8 +40,7 @@ public abstract class IndexTestContext<T extends CheckTuple> implements IIndexTe throws HyracksDataException { this.fieldSerdes = fieldSerdes; this.index = index; - this.indexAccessor = (IIndexAccessor) index.createAccessor(TestOperationCallback.INSTANCE, - TestOperationCallback.INSTANCE); + this.indexAccessor = index.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE); this.tupleBuilder = filtered ? new ArrayTupleBuilder(fieldSerdes.length + 1) : new ArrayTupleBuilder(fieldSerdes.length); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java index 5d6d8de..7718ed9 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java @@ -26,7 +26,7 @@ import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder; import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference; import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer; import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree; -import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeDiskComponent; +import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBloomFilterDiskComponent; import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext; import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; @@ -81,8 +81,9 @@ public class LSMBTreeFileManagerTest { ctx.getIndex().deactivate(); // Delete the btree file and keep the bloom filter file from the disk component - LSMBTreeDiskComponent ilsmDiskComponent = (LSMBTreeDiskComponent) btree.getDiskComponents().get(0); - ilsmDiskComponent.getBTree().getFileReference().delete(); + LSMBTreeWithBloomFilterDiskComponent ilsmDiskComponent = + (LSMBTreeWithBloomFilterDiskComponent) btree.getDiskComponents().get(0); + ilsmDiskComponent.getIndex().getFileReference().delete(); File bloomFilterFile = ilsmDiskComponent.getBloomFilter().getFileReference().getFile().getAbsoluteFile(); Assert.assertEquals("Check bloom filter file exists", true, bloomFilterFile.exists()); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java index fbcbcc2..a5c9d21 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java @@ -48,10 +48,10 @@ import org.apache.hyracks.storage.am.common.TestOperationCallback; import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor; import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig; import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree; -import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeDiskComponent; import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleReference; import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext; import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; import org.apache.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallbackFactory; import org.apache.hyracks.storage.common.IIndexCursor; @@ -172,8 +172,9 @@ public class LSMBTreeUpdateInPlaceScanDiskComponentsTest extends OrderedIndexTes (IIndexTestContext ctx, ITupleReference tuple, UpdatedCheckTuple checkTuple) -> { ctx.getIndexAccessor().delete(tuple); // Remove check tuple from expected results. - if (!checkTuples.contains(checkTuple)) + if (!checkTuples.contains(checkTuple)) { fail("Trying to delete tuple " + checkTuple + " that does not exist"); + } checkTuple.setUpdated(!checkTuple.isUpdated()); checkTuple.setAntimatter(true); }; @@ -272,8 +273,9 @@ public class LSMBTreeUpdateInPlaceScanDiskComponentsTest extends OrderedIndexTes // Copy CheckTuple references into array, so we can randomly pick from there. UpdatedCheckTuple[] checkTuplesArray = new UpdatedCheckTuple[numCheckTuples]; int idx = 0; - for (UpdatedCheckTuple t : checkTuples) + for (UpdatedCheckTuple t : checkTuples) { checkTuplesArray[idx++] = t; + } for (int i = 0; i < numTuples && numCheckTuples > 0; i++) { if (LOGGER.isLoggable(Level.INFO)) { @@ -357,8 +359,8 @@ public class LSMBTreeUpdateInPlaceScanDiskComponentsTest extends OrderedIndexTes LSMBTree btree = (LSMBTree) ctx.getIndex(); Assert.assertEquals("Check disk components", 1, btree.getDiskComponents().size()); - LSMBTreeDiskComponent btreeComponent = (LSMBTreeDiskComponent) btree.getDiskComponents().get(0); - BTree.BTreeAccessor btreeAccessor = (BTree.BTreeAccessor) btreeComponent.getBTree() + ILSMDiskComponent btreeComponent = btree.getDiskComponents().get(0); + BTree.BTreeAccessor btreeAccessor = ((BTree) btreeComponent.getIndex()) .createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE); ITreeIndexCursor cursor = btreeAccessor.createDiskOrderScanCursor(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtree.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtree.java index 6c1a406..2e0f196 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtree.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtree.java @@ -26,14 +26,13 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory; -import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.common.api.IIndexOperationContext; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree; import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; @@ -46,7 +45,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager; import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor; -import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; +import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.util.trace.ITracer; public class TestLsmBtree extends LSMBTree { @@ -70,16 +69,16 @@ public class TestLsmBtree extends LSMBTree { public TestLsmBtree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory, - ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager, - TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory, - BloomFilterFactory bloomFilterFactory, IComponentFilterHelper filterHelper, + ITreeIndexFrameFactory deleteLeafFrameFactory, IBufferCache diskBufferCache, + ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory, + ILSMDiskComponentFactory bulkLoadComponentFactory, IComponentFilterHelper filterHelper, ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, double bloomFilterFalsePositiveRate, int fieldCount, IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, int[] btreeFields, int[] filterFields, boolean durable, boolean updateAware, ITracer tracer) throws HyracksDataException { super(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, - fileManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory, filterHelper, + diskBufferCache, fileManager, componentFactory, bulkLoadComponentFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, fieldCount, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck, btreeFields, filterFields, durable, updateAware, tracer); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtreeUtil.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtreeUtil.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtreeUtil.java index 049cb59..75539ad 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtreeUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestLsmBtreeUtil.java @@ -33,9 +33,12 @@ import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory; import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree; +import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager; +import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBloomFilterDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory; import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager; @@ -79,9 +82,6 @@ public class TestLsmBtreeUtil { TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length); - BloomFilterFactory bloomFilterFactory = - needKeyDupCheck ? new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields) : null; - ComponentFilterHelper filterHelper = null; LSMComponentFilterFrameFactory filterFrameFactory = null; LSMComponentFilterManager filterManager = null; @@ -96,8 +96,21 @@ public class TestLsmBtreeUtil { ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, file, diskBTreeFactory, needKeyDupCheck); + ILSMDiskComponentFactory componentFactory; + ILSMDiskComponentFactory bulkLoadComponentFactory; + if (needKeyDupCheck) { + BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields); + componentFactory = + new LSMBTreeWithBloomFilterDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, filterHelper); + bulkLoadComponentFactory = new LSMBTreeWithBloomFilterDiskComponentFactory(bulkLoadBTreeFactory, + bloomFilterFactory, filterHelper); + } else { + componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, filterHelper); + bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, filterHelper); + } + return new TestLsmBtree(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory, - deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory, + deleteLeafFrameFactory, diskBufferCache, fileNameManager, componentFactory, bulkLoadComponentFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, typeTraits.length, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck, btreeFields, filterFields, durable, updateAware, tracer);
