http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java deleted file mode 100644 index c169d85..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hyracks.storage.am.lsm.btree.impls; - -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; -import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType; -import org.apache.hyracks.storage.common.IIndexBulkLoader; - -public class LSMBTreeBulkLoader implements IIndexBulkLoader { - private final LSMBTree lsmIndex; - private final ILSMDiskComponent component; - private final IIndexBulkLoader componentBulkLoader; - - public LSMBTreeBulkLoader(LSMBTree lsmIndex, float fillFactor, boolean verifyInput, long numElementsHint) - throws HyracksDataException { - this.lsmIndex = lsmIndex; - this.component = lsmIndex.createBulkLoadTarget(); - this.componentBulkLoader = lsmIndex.createComponentBulkLoader(component, fillFactor, verifyInput, - numElementsHint, false, true, true); - } - - @Override - public void add(ITupleReference tuple) throws HyracksDataException { - componentBulkLoader.add(tuple); - } - - @Override - public void end() throws HyracksDataException { - componentBulkLoader.end(); - if (component.getComponentSize() > 0) { - //TODO(amoudi): Ensure Bulk load follow the same lifecycle Other Operations (Flush, Merge, etc). - //then after operation should be called from harness as well - //https://issues.apache.org/jira/browse/ASTERIXDB-1764 - lsmIndex.getIOOperationCallback().afterOperation(LSMOperationType.FLUSH, null, component); - lsmIndex.getLsmHarness().addBulkLoadedComponent(component); - } - } - - @Override - public void abort() throws HyracksDataException { - componentBulkLoader.end(); - } - -} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java index 4d60ab4..a8e707f 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java @@ -18,51 +18,47 @@ */ package org.apache.hyracks.storage.am.lsm.btree.impls; -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter; +import java.util.HashSet; +import java.util.Set; + import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.common.api.IMetadataPageManager; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponent; -import org.apache.hyracks.storage.am.lsm.common.util.ComponentUtils; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex; -public class LSMBTreeDiskComponent extends AbstractLSMDiskComponent { - private final BTree btree; - private final BloomFilter bloomFilter; +public class LSMBTreeDiskComponent extends AbstractLSMDiskComponent implements ILSMDiskComponent { + protected final BTree btree; - public LSMBTreeDiskComponent(BTree btree, BloomFilter bloomFilter, ILSMComponentFilter filter) { - super((IMetadataPageManager) btree.getPageManager(), filter); + public LSMBTreeDiskComponent(AbstractLSMIndex lsmIndex, BTree btree, ILSMComponentFilter filter) { + super(lsmIndex, getMetadataPageManager(btree), filter); this.btree = btree; - this.bloomFilter = bloomFilter; } @Override - public void destroy() throws HyracksDataException { - btree.deactivate(); - btree.destroy(); - if (bloomFilter != null) { - bloomFilter.deactivate(); - bloomFilter.destroy(); - } - } - - public BTree getBTree() { + public BTree getIndex() { return btree; } - public BloomFilter getBloomFilter() { - return bloomFilter; + @Override + public BTree getMetadataHolder() { + return btree; } @Override public long getComponentSize() { - return btree.getFileReference().getFile().length() - + (bloomFilter == null ? 0 : bloomFilter.getFileReference().getFile().length()); + return getComponentSize(btree); } @Override public int getFileReferenceCount() { - return btree.getBufferCache().getFileReferenceCount(btree.getFileId()); + return getFileReferenceCount(btree); + } + + @Override + public Set<String> getLSMComponentPhysicalFiles() { + return getFiles(btree); } @Override @@ -70,13 +66,21 @@ public class LSMBTreeDiskComponent extends AbstractLSMDiskComponent { return getClass().getSimpleName() + ":" + btree.getFileReference().getRelativePath(); } - @Override - public void markAsValid(boolean persist) throws HyracksDataException { - // The order of forcing the dirty page to be flushed is critical. - // The bloom filter must be always done first. - if (bloomFilter != null && persist) { - ComponentUtils.markAsValid(btree.getBufferCache(), bloomFilter, persist); - } - ComponentUtils.markAsValid(btree, persist); + static IMetadataPageManager getMetadataPageManager(BTree btree) { + return (IMetadataPageManager) btree.getPageManager(); + } + + static long getComponentSize(BTree btree) { + return btree.getFileReference().getFile().length(); + } + + static int getFileReferenceCount(BTree btree) { + return btree.getBufferCache().getFileReferenceCount(btree.getFileId()); + } + + static Set<String> getFiles(BTree btree) { + Set<String> files = new HashSet<>(); + files.add(btree.getFileReference().getFile().getAbsolutePath()); + return files; } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java deleted file mode 100644 index 43373b5..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hyracks.storage.am.lsm.btree.impls; - -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterManager; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; -import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponentBulkLoader; -import org.apache.hyracks.storage.common.IIndex; -import org.apache.hyracks.storage.common.MultiComparator; - -public class LSMBTreeDiskComponentBulkLoader extends AbstractLSMDiskComponentBulkLoader { - - //with filter - public LSMBTreeDiskComponentBulkLoader(LSMBTreeDiskComponent component, BloomFilterSpecification bloomFilterSpec, - float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, - boolean cleanupEmptyComponent, ILSMComponentFilterManager filterManager, int[] indexFields, - int[] filterFields, MultiComparator filterCmp) throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, - cleanupEmptyComponent, filterManager, indexFields, filterFields, filterCmp); - } - - //without filter - public LSMBTreeDiskComponentBulkLoader(LSMBTreeDiskComponent component, BloomFilterSpecification bloomFilterSpec, - float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, - boolean cleanupEmptyComponent) throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, - cleanupEmptyComponent, null, null, null, null); - } - - @Override - protected BloomFilter getBloomFilter(ILSMDiskComponent component) { - return ((LSMBTreeDiskComponent) component).getBloomFilter(); - } - - @Override - protected IIndex getIndex(ILSMDiskComponent component) { - return ((LSMBTreeDiskComponent) component).getBTree(); - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java index 399904f..0e54f53 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java @@ -20,34 +20,27 @@ package org.apache.hyracks.storage.am.lsm.btree.impls; import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory; import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; public class LSMBTreeDiskComponentFactory implements ILSMDiskComponentFactory { - private final TreeIndexFactory<BTree> btreeFactory; - private final BloomFilterFactory bloomFilterFactory; - private final IComponentFilterHelper filterHelper; + protected final TreeIndexFactory<BTree> btreeFactory; + protected final IComponentFilterHelper filterHelper; - public LSMBTreeDiskComponentFactory(TreeIndexFactory<BTree> btreeFactory, BloomFilterFactory bloomFilterFactory, - IComponentFilterHelper filterHelper) { + public LSMBTreeDiskComponentFactory(TreeIndexFactory<BTree> btreeFactory, IComponentFilterHelper filterHelper) { this.btreeFactory = btreeFactory; - this.bloomFilterFactory = bloomFilterFactory; this.filterHelper = filterHelper; } @Override - public LSMBTreeDiskComponent createComponent(LSMComponentFileReferences cfr) throws HyracksDataException { - return new LSMBTreeDiskComponent(btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()), - bloomFilterFactory == null ? null - : bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()), + public LSMBTreeDiskComponent createComponent(AbstractLSMIndex lsmIndex, LSMComponentFileReferences cfr) + throws HyracksDataException { + return new LSMBTreeDiskComponent(lsmIndex, btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()), filterHelper == null ? null : filterHelper.createFilter()); } - public int[] getBloomFilterKeyFields() { - return bloomFilterFactory == null ? null : bloomFilterFactory.getBloomFilterKeyFields(); - } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java index ab9b5af..02e481c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java @@ -69,7 +69,6 @@ public class LSMBTreeDiskComponentScanCursor extends LSMIndexSearchCursor { operationalComponents = lsmInitialState.getOperationalComponents(); lsmHarness = lsmInitialState.getLSMHarness(); includeMutableComponent = false; - int numBTrees = operationalComponents.size(); rangeCursors = new IIndexCursor[numBTrees]; btreeAccessors = new BTreeAccessor[numBTrees]; @@ -77,10 +76,9 @@ public class LSMBTreeDiskComponentScanCursor extends LSMIndexSearchCursor { ILSMComponent component = operationalComponents.get(i); IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) lsmInitialState.getLeafFrameFactory().createFrame(); rangeCursors[i] = new BTreeRangeSearchCursor(leafFrame, false); - BTree btree = ((LSMBTreeDiskComponent) component).getBTree(); + BTree btree = (BTree) component.getIndex(); - btreeAccessors[i] = (BTreeAccessor) btree.createAccessor(NoOpOperationCallback.INSTANCE, - NoOpOperationCallback.INSTANCE); + btreeAccessors[i] = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); btreeAccessors[i].search(rangeCursors[i], searchPred); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java index 32b93bb..b334d80 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java @@ -19,7 +19,6 @@ package org.apache.hyracks.storage.am.lsm.btree.impls; -import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter; import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; @@ -29,23 +28,13 @@ public class LSMBTreeMemoryComponent extends AbstractLSMMemoryComponent { private final BTree btree; - public LSMBTreeMemoryComponent(BTree btree, IVirtualBufferCache vbc, boolean isActive, - ILSMComponentFilter filter) { + public LSMBTreeMemoryComponent(BTree btree, IVirtualBufferCache vbc, boolean isActive, ILSMComponentFilter filter) { super(vbc, isActive, filter); this.btree = btree; } - public BTree getBTree() { - return btree; - } - @Override - public void reset() throws HyracksDataException { - super.reset(); - btree.deactivate(); - btree.destroy(); - btree.create(); - btree.activate(); + public BTree getIndex() { + return btree; } - } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java index 86502a7..08ba7af 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java @@ -66,29 +66,29 @@ public final class LSMBTreeOpContext extends AbstractLSMIndexOperationContext { ILSMHarness lsmHarness, IBinaryComparatorFactory[] filterCmpFactories) { super(btreeFields, filterFields, filterCmpFactories, searchCallback, modificationCallback); LSMBTreeMemoryComponent c = (LSMBTreeMemoryComponent) mutableComponents.get(0); - IBinaryComparatorFactory cmpFactories[] = c.getBTree().getComparatorFactories(); + IBinaryComparatorFactory cmpFactories[] = c.getIndex().getComparatorFactories(); if (cmpFactories[0] != null) { - this.cmp = MultiComparator.create(c.getBTree().getComparatorFactories()); + this.cmp = MultiComparator.create(c.getIndex().getComparatorFactories()); } else { this.cmp = null; } bloomFilterCmp = numBloomFilterKeyFields == 0 ? null - : MultiComparator.create(c.getBTree().getComparatorFactories(), 0, numBloomFilterKeyFields); + : MultiComparator.create(c.getIndex().getComparatorFactories(), 0, numBloomFilterKeyFields); mutableBTrees = new BTree[mutableComponents.size()]; mutableBTreeAccessors = new BTree.BTreeAccessor[mutableComponents.size()]; mutableBTreeOpCtxs = new BTreeOpContext[mutableComponents.size()]; for (int i = 0; i < mutableComponents.size(); i++) { LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) mutableComponents.get(i); - mutableBTrees[i] = mutableComponent.getBTree(); + mutableBTrees[i] = mutableComponent.getIndex(); if (allFields != null) { - mutableBTreeAccessors[i] = (BTree.BTreeAccessor) mutableBTrees[i].createAccessor(modificationCallback, + mutableBTreeAccessors[i] = mutableBTrees[i].createAccessor(modificationCallback, NoOpOperationCallback.INSTANCE, allFields); } else { - mutableBTreeAccessors[i] = (BTree.BTreeAccessor) mutableBTrees[i].createAccessor(modificationCallback, - NoOpOperationCallback.INSTANCE); + mutableBTreeAccessors[i] = + mutableBTrees[i].createAccessor(modificationCallback, NoOpOperationCallback.INSTANCE); } mutableBTreeOpCtxs[i] = mutableBTreeAccessors[i].getOpContext(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java index 6126c53..c178b51 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java @@ -178,24 +178,24 @@ public class LSMBTreePointSearchCursor implements ITreeIndexCursor { // reset rangeCursors[i].reset(); } - btree = ((LSMBTreeMemoryComponent) component).getBTree(); + btree = ((LSMBTreeMemoryComponent) component).getIndex(); } else { if (rangeCursors[i] != null && rangeCursors[i].isBloomFilterAware()) { // can re-use cursor ((BloomFilterAwareBTreePointSearchCursor) rangeCursors[i]) - .resetBloomFilter(((LSMBTreeDiskComponent) component).getBloomFilter()); + .resetBloomFilter(((LSMBTreeWithBloomFilterDiskComponent) component).getBloomFilter()); rangeCursors[i].reset(); } else { // create new cursor <should be relatively rare> IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) lsmInitialState.getLeafFrameFactory().createFrame(); rangeCursors[i] = new BloomFilterAwareBTreePointSearchCursor(leafFrame, false, - ((LSMBTreeDiskComponent) component).getBloomFilter()); + ((LSMBTreeWithBloomFilterDiskComponent) component).getBloomFilter()); } - btree = ((LSMBTreeDiskComponent) component).getBTree(); + btree = (BTree) component.getIndex(); } if (btreeAccessors[i] == null) { - btreeAccessors[i] = (BTreeAccessor) btree.createAccessor(NoOpOperationCallback.INSTANCE, - NoOpOperationCallback.INSTANCE); + btreeAccessors[i] = + btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); } else { // re-use btreeAccessors[i].reset(btree, NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java index 29d7c60..7f93b62 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java @@ -231,14 +231,14 @@ public class LSMBTreeRangeSearchCursor extends LSMIndexSearchCursor { } if (component.getType() == LSMComponentType.MEMORY) { includeMutableComponent = true; - btree = ((LSMBTreeMemoryComponent) component).getBTree(); + btree = (BTree) component.getIndex(); } else { - btree = ((LSMBTreeDiskComponent) component).getBTree(); + btree = (BTree) component.getIndex(); } if (btreeAccessors[i] == null) { - btreeAccessors[i] = (BTreeAccessor) btree.createAccessor(NoOpOperationCallback.INSTANCE, - NoOpOperationCallback.INSTANCE); + btreeAccessors[i] = + btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); } else { // re-use btreeAccessors[i].reset(btree, NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBloomFilterDiskComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBloomFilterDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBloomFilterDiskComponent.java new file mode 100644 index 0000000..fa17549 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBloomFilterDiskComponent.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.storage.am.lsm.btree.impls; + +import java.util.Set; + +import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter; +import org.apache.hyracks.storage.am.btree.impls.BTree; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter; +import org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBloomFilterDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex; +import org.apache.hyracks.storage.common.buffercache.IBufferCache; + +public class LSMBTreeWithBloomFilterDiskComponent extends AbstractLSMWithBloomFilterDiskComponent { + + private final BTree btree; + private final BloomFilter bloomFilter; + + public LSMBTreeWithBloomFilterDiskComponent(AbstractLSMIndex lsmIndex, BTree btree, BloomFilter bloomFilter, + ILSMComponentFilter filter) { + super(lsmIndex, LSMBTreeDiskComponent.getMetadataPageManager(btree), filter); + this.btree = btree; + this.bloomFilter = bloomFilter; + } + + @Override + public BloomFilter getBloomFilter() { + return bloomFilter; + } + + @Override + public IBufferCache getBloomFilterBufferCache() { + return getMetadataHolder().getBufferCache(); + } + + @Override + public long getComponentSize() { + return LSMBTreeDiskComponent.getComponentSize(btree) + getComponentSize(bloomFilter); + } + + @Override + public Set<String> getLSMComponentPhysicalFiles() { + Set<String> files = LSMBTreeDiskComponent.getFiles(btree); + addFiles(files, bloomFilter); + return files; + } + + static void addFiles(Set<String> files, BloomFilter bloomFilter) { + files.add(bloomFilter.getFileReference().getFile().getAbsolutePath()); + } + + @Override + public int getFileReferenceCount() { + return LSMBTreeDiskComponent.getFileReferenceCount(btree); + } + + @Override + public BTree getMetadataHolder() { + return btree; + } + + @Override + public BTree getIndex() { + return btree; + } + + static long getComponentSize(BloomFilter bloomFilter) { + return bloomFilter.getFileReference().getFile().length(); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBloomFilterDiskComponentFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBloomFilterDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBloomFilterDiskComponentFactory.java new file mode 100644 index 0000000..09e83cc --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBloomFilterDiskComponentFactory.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.storage.am.lsm.btree.impls; + +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory; +import org.apache.hyracks.storage.am.btree.impls.BTree; +import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex; +import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; +import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; + +public class LSMBTreeWithBloomFilterDiskComponentFactory implements ILSMDiskComponentFactory { + protected final TreeIndexFactory<BTree> btreeFactory; + protected final IComponentFilterHelper filterHelper; + protected final BloomFilterFactory bloomFilterFactory; + + public LSMBTreeWithBloomFilterDiskComponentFactory(TreeIndexFactory<BTree> btreeFactory, + BloomFilterFactory bloomFilterFactory, IComponentFilterHelper filterHelper) { + this.btreeFactory = btreeFactory; + this.filterHelper = filterHelper; + this.bloomFilterFactory = bloomFilterFactory; + } + + @Override + public LSMBTreeWithBloomFilterDiskComponent createComponent(AbstractLSMIndex lsmIndex, + LSMComponentFileReferences cfr) throws HyracksDataException { + return new LSMBTreeWithBloomFilterDiskComponent(lsmIndex, + btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()), + bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()), + filterHelper == null ? null : filterHelper.createFilter()); + } + + public int[] getBloomFilterKeyFields() { + return bloomFilterFactory == null ? null : bloomFilterFactory.getBloomFilterKeyFields(); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java index 03d28d0..5122e43 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java @@ -33,6 +33,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType; import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext; +import org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBuddyDiskComponent; import org.apache.hyracks.storage.am.lsm.common.impls.BloomFilterAwareBTreePointSearchCursor; import org.apache.hyracks.storage.common.ICursorInitialState; import org.apache.hyracks.storage.common.ISearchPredicate; @@ -104,26 +105,26 @@ public abstract class LSMBTreeWithBuddyAbstractCursor implements ITreeIndexCurso } else { buddyBtreeCursors[i].reset(); } - btree = ((LSMBTreeWithBuddyMemoryComponent) component).getBTree(); - buddyBtree = ((LSMBTreeWithBuddyMemoryComponent) component).getBuddyBTree(); + btree = ((LSMBTreeWithBuddyMemoryComponent) component).getIndex(); + buddyBtree = ((LSMBTreeWithBuddyMemoryComponent) component).getBuddyIndex(); } else { if (buddyBtreeCursors[i] == null || !buddyBtreeCursors[i].isBloomFilterAware()) { buddyBtreeCursors[i] = new BloomFilterAwareBTreePointSearchCursor( (IBTreeLeafFrame) lsmInitialState.getBuddyBTreeLeafFrameFactory().createFrame(), false, - ((LSMBTreeWithBuddyDiskComponent) operationalComponents.get(i)).getBloomFilter()); + ((AbstractLSMWithBuddyDiskComponent) operationalComponents.get(i)).getBloomFilter()); } else { buddyBtreeCursors[i].reset(); } - btree = ((LSMBTreeWithBuddyDiskComponent) component).getBTree(); - buddyBtree = ((LSMBTreeWithBuddyDiskComponent) component).getBuddyBTree(); + btree = (BTree) component.getIndex(); + buddyBtree = (BTree) ((AbstractLSMWithBuddyDiskComponent) component).getBuddyIndex(); } IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) lsmInitialState.getBTreeLeafFrameFactory().createFrame(); if (btreeAccessors[i] == null) { btreeCursors[i] = new BTreeRangeSearchCursor(leafFrame, false); - btreeAccessors[i] = (BTreeAccessor) btree.createAccessor(NoOpOperationCallback.INSTANCE, - NoOpOperationCallback.INSTANCE); - buddyBtreeAccessors[i] = (BTreeAccessor) buddyBtree.createAccessor(NoOpOperationCallback.INSTANCE, - NoOpOperationCallback.INSTANCE); + btreeAccessors[i] = + btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); + buddyBtreeAccessors[i] = + buddyBtree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); } else { btreeCursors[i].reset(); btreeAccessors[i].reset(btree, NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java index 4aee950..1e9dade 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java @@ -18,72 +18,76 @@ */ package org.apache.hyracks.storage.am.lsm.btree.impls; +import java.util.Set; + import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter; import org.apache.hyracks.storage.am.btree.impls.BTree; -import org.apache.hyracks.storage.am.common.api.IMetadataPageManager; +import org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBuddyDiskComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter; -import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponent; -import org.apache.hyracks.storage.am.lsm.common.util.ComponentUtils; - -public class LSMBTreeWithBuddyDiskComponent extends AbstractLSMDiskComponent { +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex; +import org.apache.hyracks.storage.common.buffercache.IBufferCache; +public class LSMBTreeWithBuddyDiskComponent extends AbstractLSMWithBuddyDiskComponent { private final BTree btree; - private final BTree buddyBtree; private final BloomFilter bloomFilter; + private final BTree buddyBtree; - public LSMBTreeWithBuddyDiskComponent(BTree btree, BTree buddyBtree, BloomFilter bloomFilter, - ILSMComponentFilter filter) { - super((IMetadataPageManager) btree.getPageManager(), filter); + public LSMBTreeWithBuddyDiskComponent(AbstractLSMIndex lsmIndex, BTree btree, BTree buddyBtree, + BloomFilter bloomFilter, ILSMComponentFilter filter) { + super(lsmIndex, LSMBTreeDiskComponent.getMetadataPageManager(btree), filter); this.btree = btree; - this.buddyBtree = buddyBtree; this.bloomFilter = bloomFilter; + this.buddyBtree = buddyBtree; } @Override - public void destroy() throws HyracksDataException { - btree.deactivate(); - btree.destroy(); - buddyBtree.deactivate(); - buddyBtree.destroy(); - bloomFilter.deactivate(); - bloomFilter.destroy(); + public BTree getBuddyIndex() { + return buddyBtree; } - public BTree getBTree() { - return btree; + @Override + public long getComponentSize() { + return LSMBTreeDiskComponent.getComponentSize(btree) + + LSMBTreeWithBloomFilterDiskComponent.getComponentSize(bloomFilter) + + buddyBtree.getFileReference().getFile().length(); } - public BTree getBuddyBTree() { - return buddyBtree; + @Override + public Set<String> getLSMComponentPhysicalFiles() { + Set<String> files = LSMBTreeDiskComponent.getFiles(btree); + LSMBTreeWithBloomFilterDiskComponent.addFiles(files, bloomFilter); + files.add(buddyBtree.getFileReference().getFile().getAbsolutePath()); + return files; } - public BloomFilter getBloomFilter() { - return bloomFilter; + @Override + public void validate() throws HyracksDataException { + throw new UnsupportedOperationException("Validation not implemented for LSM B-Trees with Buddy B-Tree."); } @Override - public long getComponentSize() { - long size = btree.getFileReference().getFile().length(); - size += buddyBtree.getFileReference().getFile().length(); - size += bloomFilter.getFileReference().getFile().length(); - return size; + public int getFileReferenceCount() { + return LSMBTreeDiskComponent.getFileReferenceCount(btree); } @Override - public int getFileReferenceCount() { - return btree.getBufferCache().getFileReferenceCount(btree.getFileId()); + public BTree getMetadataHolder() { + return btree; } @Override - public String toString() { - return getClass().getSimpleName() + ":" + btree.getFileReference().getRelativePath(); + public BTree getIndex() { + return btree; + } + + @Override + public BloomFilter getBloomFilter() { + return bloomFilter; } @Override - public void markAsValid(boolean persist) throws HyracksDataException { - ComponentUtils.markAsValid(btree.getBufferCache(), bloomFilter, persist); - ComponentUtils.markAsValid(btree, persist); - ComponentUtils.markAsValid(buddyBtree, persist); + public IBufferCache getBloomFilterBufferCache() { + return getMetadataHolder().getBufferCache(); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java deleted file mode 100644 index b3e7c6c..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hyracks.storage.am.lsm.btree.impls; - -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter; -import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification; -import org.apache.hyracks.storage.am.common.api.ITreeIndex; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterManager; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; -import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponentWithBuddyBulkLoader; -import org.apache.hyracks.storage.common.IIndex; -import org.apache.hyracks.storage.common.MultiComparator; - -public class LSMBTreeWithBuddyDiskComponentBulkLoader extends AbstractLSMDiskComponentWithBuddyBulkLoader { - - //with filter - public LSMBTreeWithBuddyDiskComponentBulkLoader(LSMBTreeWithBuddyDiskComponent component, - BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint, - boolean checkIfEmptyIndex, boolean cleanupEmptyComponent, ILSMComponentFilterManager filterManager, - int[] indexFields, int[] filterFields, MultiComparator filterCmp) throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, - cleanupEmptyComponent, filterManager, indexFields, filterFields, filterCmp); - } - - //without filter - public LSMBTreeWithBuddyDiskComponentBulkLoader(LSMBTreeWithBuddyDiskComponent component, - BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint, - boolean checkIfEmptyIndex, boolean cleanupEmptyComponent) throws HyracksDataException { - super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, - cleanupEmptyComponent, null, null, null, null); - } - - @Override - protected BloomFilter getBloomFilter(ILSMDiskComponent component) { - return ((LSMBTreeWithBuddyDiskComponent) component).getBloomFilter(); - } - - @Override - protected IIndex getIndex(ILSMDiskComponent component) { - return ((LSMBTreeWithBuddyDiskComponent) component).getBTree(); - } - - @Override - protected ITreeIndex getBuddyBTree(ILSMDiskComponent component) { - return ((LSMBTreeWithBuddyDiskComponent) component).getBuddyBTree(); - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentFactory.java index 4223954..bfa2314 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentFactory.java @@ -21,27 +21,39 @@ package org.apache.hyracks.storage.am.lsm.btree.impls; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory; import org.apache.hyracks.storage.am.btree.impls.BTree; +import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper; import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; public class LSMBTreeWithBuddyDiskComponentFactory implements ILSMDiskComponentFactory { - private final TreeIndexFactory<BTree> btreeFactory; - private final TreeIndexFactory<BTree> buddyBtreeFactory; - private final BloomFilterFactory bloomFilterFactory; + protected final TreeIndexFactory<BTree> btreeFactory; + protected final IComponentFilterHelper filterHelper; + protected final BloomFilterFactory bloomFilterFactory; + protected final TreeIndexFactory<BTree> buddyBtreeFactory; public LSMBTreeWithBuddyDiskComponentFactory(TreeIndexFactory<BTree> btreeFactory, - TreeIndexFactory<BTree> buddyBtreeFactory, BloomFilterFactory bloomFilterFactory) { + TreeIndexFactory<BTree> buddyBtreeFactory, BloomFilterFactory bloomFilterFactory, + IComponentFilterHelper filterHelper) { this.btreeFactory = btreeFactory; - this.buddyBtreeFactory = buddyBtreeFactory; + this.filterHelper = filterHelper; this.bloomFilterFactory = bloomFilterFactory; + this.buddyBtreeFactory = buddyBtreeFactory; + } + + public int[] getBloomFilterKeyFields() { + return bloomFilterFactory == null ? null : bloomFilterFactory.getBloomFilterKeyFields(); } @Override - public LSMBTreeWithBuddyDiskComponent createComponent(LSMComponentFileReferences cfr) throws HyracksDataException { - return new LSMBTreeWithBuddyDiskComponent(btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()), + public LSMBTreeWithBuddyDiskComponent createComponent(AbstractLSMIndex lsmIndex, LSMComponentFileReferences cfr) + throws HyracksDataException { + return new LSMBTreeWithBuddyDiskComponent(lsmIndex, + btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()), buddyBtreeFactory.createIndexInstance(cfr.getDeleteIndexFileReference()), - bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()), null); + bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()), + filterHelper == null ? null : filterHelper.createFilter()); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMemoryComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMemoryComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMemoryComponent.java index f58c737..828b58e 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMemoryComponent.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMemoryComponent.java @@ -21,13 +21,13 @@ package org.apache.hyracks.storage.am.lsm.btree.impls; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter; +import org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBuddyMemoryComponent; import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; -import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMMemoryComponent; /* * This class is also not needed at the moment but is implemented anyway */ -public class LSMBTreeWithBuddyMemoryComponent extends AbstractLSMMemoryComponent { +public class LSMBTreeWithBuddyMemoryComponent extends AbstractLSMWithBuddyMemoryComponent { private final BTree btree; private final BTree buddyBtree; @@ -39,25 +39,23 @@ public class LSMBTreeWithBuddyMemoryComponent extends AbstractLSMMemoryComponent this.buddyBtree = buddyBtree; } - public BTree getBTree() { + @Override + public BTree getIndex() { return btree; } - public BTree getBuddyBTree() { + @Override + public BTree getBuddyIndex() { return buddyBtree; } @Override - public void reset() throws HyracksDataException { - super.reset(); - btree.deactivate(); - btree.destroy(); - btree.create(); - btree.activate(); - buddyBtree.deactivate(); - buddyBtree.destroy(); - buddyBtree.create(); - buddyBtree.activate(); + public void validate() throws HyracksDataException { + throw new UnsupportedOperationException("Validation not implemented for LSM B-Trees with Buddy B-Tree."); } + @Override + public long getSize() { + return 0L; + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBuddyBTreeMergeCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBuddyBTreeMergeCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBuddyBTreeMergeCursor.java index 79d6b0f..061cad5 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBuddyBTreeMergeCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBuddyBTreeMergeCursor.java @@ -62,7 +62,7 @@ public class LSMBuddyBTreeMergeCursor extends LSMIndexSearchCursor { ILSMComponent component = operationalComponents.get(i); IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) lsmInitialState.getBuddyBTreeLeafFrameFactory().createFrame(); rangeCursors[i] = new BTreeRangeSearchCursor(leafFrame, false); - BTree buddyBtree = ((LSMBTreeWithBuddyDiskComponent) component).getBuddyBTree(); + BTree buddyBtree = ((LSMBTreeWithBuddyDiskComponent) component).getBuddyIndex(); btreeAccessors[i] = buddyBtree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); btreeAccessors[i].search(rangeCursors[i], btreePredicate); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java index 4aafbe9..2c4e6b1 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java @@ -37,10 +37,14 @@ import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory; import org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree; import org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTreeWithBuddy; import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree; +import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager; +import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBloomFilterDiskComponentFactory; +import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyFileManager; import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory; import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager; @@ -89,9 +93,6 @@ public class LSMBTreeUtil { new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, interiorFrameFactory, bulkLoadLeafFrameFactory, cmpFactories, typeTraits.length); - BloomFilterFactory bloomFilterFactory = - needKeyDupCheck ? new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields) : null; - ComponentFilterHelper filterHelper = null; LSMComponentFilterFrameFactory filterFrameFactory = null; LSMComponentFilterManager filterManager = null; @@ -106,8 +107,21 @@ public class LSMBTreeUtil { ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, file, diskBTreeFactory, needKeyDupCheck); + ILSMDiskComponentFactory componentFactory; + ILSMDiskComponentFactory bulkLoadComponentFactory; + if (needKeyDupCheck) { + BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields); + componentFactory = + new LSMBTreeWithBloomFilterDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, filterHelper); + bulkLoadComponentFactory = new LSMBTreeWithBloomFilterDiskComponentFactory(bulkLoadBTreeFactory, + bloomFilterFactory, filterHelper); + } else { + componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, filterHelper); + bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, filterHelper); + } + return new LSMBTree(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory, - deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory, + deleteLeafFrameFactory, diskBufferCache, fileNameManager, componentFactory, bulkLoadComponentFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, typeTraits.length, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck, btreeFields, filterFields, durable, updateAware, tracer); @@ -149,11 +163,19 @@ public class LSMBTreeUtil { transactionLeafFrameFactory, cmpFactories, typeTraits.length); //TODO remove BloomFilter from external dataset's secondary LSMBTree index ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, file, diskBTreeFactory, true); + + ILSMDiskComponentFactory componentFactory = + new LSMBTreeWithBloomFilterDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, null); + ILSMDiskComponentFactory bulkLoadComponentFactory = + new LSMBTreeWithBloomFilterDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory, null); + ILSMDiskComponentFactory transactionComponentFactory = + new LSMBTreeWithBloomFilterDiskComponentFactory(transactionBTreeFactory, bloomFilterFactory, null); + // the disk only index uses an empty ArrayList for virtual buffer caches return new ExternalBTree(ioManager, interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, - fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory, - bloomFilterFalsePositiveRate, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, - transactionBTreeFactory, durable); + diskBufferCache, fileNameManager, componentFactory, bulkLoadComponentFactory, + transactionComponentFactory, bloomFilterFalsePositiveRate, cmpFactories, mergePolicy, opTracker, + ioScheduler, ioOpCallback, durable); } public static ExternalBTreeWithBuddy createExternalBTreeWithBuddy(IIOManager ioManager, FileReference file, @@ -200,12 +222,15 @@ public class LSMBTreeUtil { ILSMIndexFileManager fileNameManager = new LSMBTreeWithBuddyFileManager(ioManager, file, diskBTreeFactory, diskBuddyBTreeFactory); + ILSMDiskComponentFactory componentFactory = new LSMBTreeWithBuddyDiskComponentFactory(diskBTreeFactory, + diskBuddyBTreeFactory, bloomFilterFactory, null); + ILSMDiskComponentFactory bulkLoadComponentFactory = new LSMBTreeWithBuddyDiskComponentFactory( + bulkLoadBTreeFactory, diskBuddyBTreeFactory, bloomFilterFactory, null); + // the disk only index uses an empty ArrayList for virtual buffer caches - ExternalBTreeWithBuddy lsmTree = new ExternalBTreeWithBuddy(ioManager, interiorFrameFactory, - insertLeafFrameFactory, buddyBtreeLeafFrameFactory, diskBufferCache, fileNameManager, - bulkLoadBTreeFactory, diskBTreeFactory, diskBuddyBTreeFactory, bloomFilterFactory, - bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, cmpFactories, - buddyBtreeCmpFactories, buddyBTreeFields, durable); - return lsmTree; + return new ExternalBTreeWithBuddy(ioManager, interiorFrameFactory, insertLeafFrameFactory, + buddyBtreeLeafFrameFactory, diskBufferCache, fileNameManager, componentFactory, + bulkLoadComponentFactory, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, + ioOpCallback, cmpFactories, buddyBtreeCmpFactories, buddyBTreeFields, durable); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBloomFilterDiskComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBloomFilterDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBloomFilterDiskComponent.java new file mode 100644 index 0000000..107190d --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBloomFilterDiskComponent.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.storage.am.lsm.common.api; + +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations; +import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter; +import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification; +import org.apache.hyracks.storage.am.common.api.IMetadataPageManager; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex; +import org.apache.hyracks.storage.am.lsm.common.impls.BloomFilterBulkLoader; +import org.apache.hyracks.storage.am.lsm.common.impls.ChainedLSMDiskComponentBulkLoader; +import org.apache.hyracks.storage.am.lsm.common.impls.IChainedComponentBulkLoader; +import org.apache.hyracks.storage.am.lsm.common.util.ComponentUtils; +import org.apache.hyracks.storage.common.buffercache.IBufferCache; + +public abstract class AbstractLSMWithBloomFilterDiskComponent extends AbstractLSMDiskComponent { + public AbstractLSMWithBloomFilterDiskComponent(AbstractLSMIndex lsmIndex, IMetadataPageManager mdPageManager, + ILSMComponentFilter filter) { + super(lsmIndex, mdPageManager, filter); + } + + public abstract BloomFilter getBloomFilter(); + + public abstract IBufferCache getBloomFilterBufferCache(); + + @Override + public void markAsValid(boolean persist) throws HyracksDataException { + // The order of forcing the dirty page to be flushed is critical. The + // bloom filter must be always done first. + ComponentUtils.markAsValid(getBloomFilterBufferCache(), getBloomFilter(), persist); + super.markAsValid(persist); + } + + @Override + public void activate(boolean createNewComponent) throws HyracksDataException { + super.activate(createNewComponent); + if (createNewComponent) { + getBloomFilter().create(); + } + getBloomFilter().activate(); + } + + @Override + public void deactivateAndDestroy() throws HyracksDataException { + super.deactivateAndDestroy(); + getBloomFilter().deactivate(); + getBloomFilter().destroy(); + } + + @Override + public void destroy() throws HyracksDataException { + super.destroy(); + getBloomFilter().destroy(); + } + + @Override + public void deactivate() throws HyracksDataException { + super.deactivate(); + getBloomFilter().deactivate(); + } + + @Override + public void deactivateAndPurge() throws HyracksDataException { + super.deactivateAndPurge(); + getBloomFilter().deactivate(); + getBloomFilter().purge(); + } + + public IChainedComponentBulkLoader createBloomFilterBulkLoader(long numElementsHint) throws HyracksDataException { + BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec( + BloomCalculations.maxBucketsPerElement(numElementsHint), getLsmIndex().bloomFilterFalsePositiveRate()); + return new BloomFilterBulkLoader(getBloomFilter().createBuilder(numElementsHint, bloomFilterSpec.getNumHashes(), + bloomFilterSpec.getNumBucketsPerElements())); + } + + @Override + public ChainedLSMDiskComponentBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, + long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter, boolean cleanupEmptyComponent) + throws HyracksDataException { + ChainedLSMDiskComponentBulkLoader chainedBulkLoader = super.createBulkLoader(fillFactor, verifyInput, + numElementsHint, checkIfEmptyIndex, withFilter, cleanupEmptyComponent); + if (numElementsHint > 0) { + chainedBulkLoader.addBulkLoader(createBloomFilterBulkLoader(numElementsHint)); + } + return chainedBulkLoader; + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBuddyDiskComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBuddyDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBuddyDiskComponent.java new file mode 100644 index 0000000..c2f52e0 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBuddyDiskComponent.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.storage.am.lsm.common.api; + +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.storage.am.common.api.IMetadataPageManager; +import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndex; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex; +import org.apache.hyracks.storage.am.lsm.common.impls.IChainedComponentBulkLoader; +import org.apache.hyracks.storage.am.lsm.common.impls.IndexWithBuddyBulkLoader; +import org.apache.hyracks.storage.am.lsm.common.util.ComponentUtils; +import org.apache.hyracks.storage.common.IIndexBulkLoader; + +public abstract class AbstractLSMWithBuddyDiskComponent extends AbstractLSMWithBloomFilterDiskComponent { + + public AbstractLSMWithBuddyDiskComponent(AbstractLSMIndex lsmIndex, IMetadataPageManager mdPageManager, + ILSMComponentFilter filter) { + super(lsmIndex, mdPageManager, filter); + } + + public abstract AbstractTreeIndex getBuddyIndex(); + + @Override + public void markAsValid(boolean persist) throws HyracksDataException { + super.markAsValid(persist); + ComponentUtils.markAsValid(getBuddyIndex(), persist); + } + + @Override + public void activate(boolean createNewComponent) throws HyracksDataException { + super.activate(createNewComponent); + if (createNewComponent) { + getBuddyIndex().create(); + } + getBuddyIndex().activate(); + } + + @Override + public void deactivateAndDestroy() throws HyracksDataException { + super.deactivateAndDestroy(); + getBuddyIndex().deactivate(); + getBuddyIndex().destroy(); + } + + @Override + public void destroy() throws HyracksDataException { + super.destroy(); + getBuddyIndex().destroy(); + } + + @Override + public void deactivate() throws HyracksDataException { + super.deactivate(); + getBuddyIndex().deactivate(); + } + + @Override + public void deactivateAndPurge() throws HyracksDataException { + super.deactivateAndPurge(); + getBuddyIndex().deactivate(); + getBuddyIndex().purge(); + } + + @Override + public void validate() throws HyracksDataException { + super.validate(); + getBuddyIndex().validate(); + } + + @Override + public IChainedComponentBulkLoader createIndexBulkLoader(float fillFactor, boolean verifyInput, + long numElementsHint, boolean checkIfEmptyIndex) throws HyracksDataException { + IIndexBulkLoader indexBulkLoader = + getIndex().createBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex); + IIndexBulkLoader buddyBulkLoader = + getBuddyIndex().createBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex); + return new IndexWithBuddyBulkLoader(indexBulkLoader, buddyBulkLoader); + } + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBuddyMemoryComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBuddyMemoryComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBuddyMemoryComponent.java new file mode 100644 index 0000000..b445142 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBuddyMemoryComponent.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.storage.am.lsm.common.api; + +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndex; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMMemoryComponent; + +public abstract class AbstractLSMWithBuddyMemoryComponent extends AbstractLSMMemoryComponent { + + public AbstractLSMWithBuddyMemoryComponent(IVirtualBufferCache vbc, boolean isActive, ILSMComponentFilter filter) { + super(vbc, isActive, filter); + } + + public abstract AbstractTreeIndex getBuddyIndex(); + + @Override + public void doReset() throws HyracksDataException { + super.doReset(); + getBuddyIndex().deactivate(); + getBuddyIndex().destroy(); + getBuddyIndex().create(); + getBuddyIndex().activate(); + } + + @Override + public void doAllocate() throws HyracksDataException { + super.doAllocate(); + getBuddyIndex().create(); + getBuddyIndex().activate(); + } + + @Override + public void doDeallocate() throws HyracksDataException { + super.doDeallocate(); + getBuddyIndex().deactivate(); + getBuddyIndex().destroy(); + } + + @Override + public void validate() throws HyracksDataException { + super.validate(); + getBuddyIndex().validate(); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponent.java index a446f4e..a60f544 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponent.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponent.java @@ -19,6 +19,7 @@ package org.apache.hyracks.storage.am.lsm.common.api; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.storage.common.IIndex; /** * An LSM index component. can be an in memory or on disk. Can be readable or unreadable, writable or unwritable @@ -121,4 +122,9 @@ public interface ILSMComponent { * @return the component filter */ ILSMComponentFilter getLSMComponentFilter(); + + /** + * @return index data structure that is the stored in the component + */ + IIndex getIndex(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java index b101315..43c5482 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java @@ -18,8 +18,14 @@ */ package org.apache.hyracks.storage.am.lsm.common.api; +import java.util.Set; + import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.storage.am.common.api.ITreeIndex; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex; +import org.apache.hyracks.storage.am.lsm.common.impls.ChainedLSMDiskComponentBulkLoader; import org.apache.hyracks.storage.am.lsm.common.impls.DiskComponentMetadata; +import org.apache.hyracks.storage.am.lsm.common.impls.IChainedComponentBulkLoader; public interface ILSMDiskComponent extends ILSMComponent { @@ -42,13 +48,6 @@ public interface ILSMDiskComponent extends ILSMComponent { int getFileReferenceCount(); /** - * Delete the component from disk - * - * @throws HyracksDataException - */ - void destroy() throws HyracksDataException; - - /** * Return the component Id of this disk component from its metadata * * @return @@ -57,6 +56,21 @@ public interface ILSMDiskComponent extends ILSMComponent { ILSMDiskComponentId getComponentId() throws HyracksDataException; /** + * @return LsmIndex of the component + */ + AbstractLSMIndex getLsmIndex(); + + /** + * @return the TreeIndex which holds metadata for the disk component + */ + ITreeIndex getMetadataHolder(); + + /** + * @return a set of files describing the contents of the disk component + */ + Set<String> getLSMComponentPhysicalFiles(); + + /** * Mark the component as valid * * @param persist @@ -65,4 +79,86 @@ public interface ILSMDiskComponent extends ILSMComponent { */ void markAsValid(boolean persist) throws HyracksDataException; + /** + * Activates the component + * + * @param create + * whether a new component should be created + * @throws HyracksDataException + */ + void activate(boolean create) throws HyracksDataException; + + /** + * Deactivate and destroy the component (Deletes it from disk) + * + * @throws HyracksDataException + */ + void deactivateAndDestroy() throws HyracksDataException; + + /** + * Destroy the component (Deletes it from disk) + * + * @throws HyracksDataException + */ + void destroy() throws HyracksDataException; + + /** + * Deactivate the component + * The pages are still in the buffer cache + * + * @throws HyracksDataException + */ + void deactivate() throws HyracksDataException; + + /** + * Deactivate the component and purge it out of the buffer cache + * + * @throws HyracksDataException + */ + void deactivateAndPurge() throws HyracksDataException; + + /** + * Test method. validates the content of the component + * TODO: Remove this method from the interface + * + * @throws HyracksDataException + */ + void validate() throws HyracksDataException; + + /** + * Creates a chained bulkloader which populates component's LSM filter + * + * @return + * @throws HyracksDataException + */ + IChainedComponentBulkLoader createFilterBulkLoader() throws HyracksDataException; + + /** + * Creates a chained bulkloader which populates component's index + * + * @param fillFactor + * @param verifyInput + * @param numElementsHint + * @param checkIfEmptyIndex + * @return + * @throws HyracksDataException + */ + IChainedComponentBulkLoader createIndexBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint, + boolean checkIfEmptyIndex) throws HyracksDataException; + + /** + * Creates a bulkloader pipeline which includes all chained operations, bulkloading individual elements of the + * component: indexes, LSM filters, Bloom filters, buddy indexes, etc. + * + * @param fillFactor + * @param verifyInput + * @param numElementsHint + * @param checkIfEmptyIndex + * @param withFilter + * @param cleanupEmptyComponent + * @return + * @throws HyracksDataException + */ + ChainedLSMDiskComponentBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint, + boolean checkIfEmptyIndex, boolean withFilter, boolean cleanupEmptyComponent) throws HyracksDataException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java index f1d1ce1..13ce971 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java @@ -34,4 +34,11 @@ public interface ILSMDiskComponentBulkLoader extends IIndexBulkLoader { */ void delete(ITupleReference tuple) throws HyracksDataException; + /** + * Releases all resources allocated during the bulkloading process + * + * @throws HyracksDataException + */ + void cleanupArtifacts() throws HyracksDataException; + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentFactory.java index 9daf30b..a904670 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentFactory.java @@ -19,6 +19,7 @@ package org.apache.hyracks.storage.am.lsm.common.api; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; @FunctionalInterface @@ -32,5 +33,6 @@ public interface ILSMDiskComponentFactory { * @return a disk component * @throws HyracksDataException */ - ILSMDiskComponent createComponent(LSMComponentFileReferences cfr) throws HyracksDataException; + ILSMDiskComponent createComponent(AbstractLSMIndex lsmIndex, LSMComponentFileReferences cfr) + throws HyracksDataException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java index addeb27..e4a9436 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java @@ -134,23 +134,6 @@ public interface ILSMIndex extends IIndex { void updateFilter(ILSMIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException; /** - * Create a component bulk loader for the given component - * - * @param component - * @param fillFactor - * @param verifyInput - * @param numElementsHint - * @param checkIfEmptyIndex - * @param withFilter - * @param cleanupEmptyComponent - * @return - * @throws HyracksDataException - */ - ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor, - boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter, - boolean cleanupEmptyComponent) throws HyracksDataException; - - /** * Creates a disk component for the bulk load operation * * @return
