Ildar Absalyamov has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/453
Change subject: Fixed m2e lifecycle mapping pom issues
......................................................................
Fixed m2e lifecycle mapping pom issues
Change-Id: Ifc8927a7fe90577d787c3d30c371ea95dd4aa24e
---
M
hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/api/INumeric.java
M
hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
M
hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
M
hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
M
hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
M
hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
M
hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java
M
hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java
M
hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
M
hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java
M
hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
M
hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java
M
hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractDiskLSMComponent.java
M
hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
M
hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java
M
hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
M
hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java
M
hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
M
hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
M
hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
M
hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
M
hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
M
hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java
M
hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
M
hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
M hyracks/pom.xml
26 files changed, 345 insertions(+), 584 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/hyracks refs/changes/53/453/1
diff --git
a/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/api/INumeric.java
b/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/api/INumeric.java
index 7be6680..a740d1a 100644
---
a/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/api/INumeric.java
+++
b/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/api/INumeric.java
@@ -30,4 +30,6 @@
public float floatValue();
public double doubleValue();
+
+ // public void shiftRight(int pos);
}
\ No newline at end of file
diff --git
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index 3ade4fa..246ea7a 100644
---
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -59,6 +59,7 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.ITwoPCIndex;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractDiskLSMComponent;
import
org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
import org.apache.hyracks.storage.am.lsm.common.impls.ExternalIndexHarness;
import
org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
@@ -97,10 +98,10 @@
ILSMIOOperationCallback ioOpCallback, TreeIndexFactory<BTree>
transactionBTreeFactory, int version,
boolean durable) {
super(interiorFrameFactory, insertLeafFrameFactory,
deleteLeafFrameFactory, fileManager, diskBTreeFactory,
- bulkLoadBTreeFactory, bloomFilterFactory,
bloomFilterFalsePositiveRate, diskFileMapProvider,
- fieldCount, cmpFactories, mergePolicy, opTracker, ioScheduler,
ioOpCallback, false, durable);
- this.transactionComponentFactory = new
LSMBTreeDiskComponentFactory(transactionBTreeFactory,
- bloomFilterFactory, null);
+ bulkLoadBTreeFactory, bloomFilterFactory,
bloomFilterFalsePositiveRate, diskFileMapProvider, fieldCount,
+ cmpFactories, mergePolicy, opTracker, ioScheduler,
ioOpCallback, false, durable);
+ this.transactionComponentFactory = new
LSMBTreeDiskComponentFactory(transactionBTreeFactory, bloomFilterFactory,
+ null);
this.secondDiskComponents = new LinkedList<ILSMComponent>();
this.interiorFrameFactory = interiorFrameFactory;
this.version = version;
@@ -116,7 +117,7 @@
throw new HyracksDataException("Failed to create transaction
components", e);
}
return createDiskComponent(transactionComponentFactory,
componentFileRefs.getInsertIndexFileReference(),
- componentFileRefs.getBloomFilterFileReference(), true);
+ componentFileRefs.getBloomFilterFileReference(),
componentFileRefs.getStatisticsFileReference(), true);
}
// The subsume merged components is overridden to account for:
@@ -180,8 +181,8 @@
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
boolean returnDeletedTuples = false;
if (version == 0) {
- if (ctx.getComponentHolder().get(ctx.getComponentHolder().size() -
1) != diskComponents.get(diskComponents
- .size() - 1)) {
+ if (ctx.getComponentHolder().get(ctx.getComponentHolder().size() -
1) != diskComponents
+ .get(diskComponents.size() - 1)) {
returnDeletedTuples = true;
}
} else {
@@ -195,12 +196,12 @@
BTree lastBTree = ((LSMBTreeDiskComponent)
mergingComponents.get(mergingComponents.size() - 1)).getBTree();
FileReference firstFile =
diskFileMapProvider.lookupFileName(firstBTree.getFileId());
FileReference lastFile =
diskFileMapProvider.lookupFileName(lastBTree.getFileId());
- LSMComponentFileReferences relMergeFileRefs =
fileManager.getRelMergeFileReference(firstFile.getFile()
- .getName(), lastFile.getFile().getName());
+ LSMComponentFileReferences relMergeFileRefs = fileManager
+ .getRelMergeFileReference(firstFile.getFile().getName(),
lastFile.getFile().getName());
ILSMIndexAccessorInternal accessor = new LSMBTreeAccessor(lsmHarness,
opCtx);
- ioScheduler.scheduleOperation(new LSMBTreeMergeOperation(accessor,
mergingComponents, cursor, relMergeFileRefs
- .getInsertIndexFileReference(),
relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager
- .getBaseDir()));
+ ioScheduler.scheduleOperation(new LSMBTreeMergeOperation(accessor,
mergingComponents, cursor,
+ relMergeFileRefs.getInsertIndexFileReference(),
relMergeFileRefs.getBloomFilterFileReference(),
+ relMergeFileRefs.getStatisticsFileReference(), callback,
fileManager.getBaseDir()));
}
// This function should only be used when a transaction fail. it doesn't
@@ -260,11 +261,12 @@
throw new HyracksDataException(e);
}
for (LSMComponentFileReferences lsmComonentFileReference :
validFileReferences) {
- LSMBTreeDiskComponent component;
+ AbstractDiskLSMComponent component;
try {
component = createDiskComponent(componentFactory,
lsmComonentFileReference.getInsertIndexFileReference(),
-
lsmComonentFileReference.getBloomFilterFileReference(), false);
+
lsmComonentFileReference.getBloomFilterFileReference(),
+
lsmComonentFileReference.getStatisticsFileReference(), false);
} catch (IndexException e) {
throw new HyracksDataException(e);
}
@@ -515,7 +517,7 @@
int maxBucketsPerElement =
BloomCalculations.maxBucketsPerElement(numElementsHint);
BloomFilterSpecification bloomFilterSpec =
BloomCalculations.computeBloomSpec(maxBucketsPerElement,
bloomFilterFalsePositiveRate);
- builder = ((LSMBTreeDiskComponent)
component).getBloomFilter().createBuilder(numElementsHint,
+ builder = ((AbstractDiskLSMComponent)
component).getBloomFilter().createBuilder(numElementsHint,
bloomFilterSpec.getNumHashes(),
bloomFilterSpec.getNumBucketsPerElements());
}
@@ -552,11 +554,11 @@
}
((LSMBTreeDiskComponent) component).getBTree().destroy();
try {
- ((LSMBTreeDiskComponent)
component).getBloomFilter().deactivate();
+ ((AbstractDiskLSMComponent)
component).getBloomFilter().deactivate();
} catch (HyracksDataException e) {
// Do nothing.. this could've bee
}
- ((LSMBTreeDiskComponent) component).getBloomFilter().destroy();
+ ((AbstractDiskLSMComponent)
component).getBloomFilter().destroy();
}
}
@@ -575,7 +577,7 @@
// deactivate. it could later be added or deleted
markAsValid(component);
BTree btree = ((LSMBTreeDiskComponent)
component).getBTree();
- BloomFilter bloomFilter = ((LSMBTreeDiskComponent)
component).getBloomFilter();
+ BloomFilter bloomFilter = ((AbstractDiskLSMComponent)
component).getBloomFilter();
btree.deactivate();
bloomFilter.deactivate();
} else {
@@ -678,11 +680,12 @@
@Override
public void commitTransaction() throws TreeIndexException,
HyracksDataException, IndexException {
- LSMComponentFileReferences componentFileRefrences =
fileManager.getTransactionFileReferenceForCommit();
- LSMBTreeDiskComponent component = null;
- if (componentFileRefrences != null) {
- component = createDiskComponent(componentFactory,
componentFileRefrences.getInsertIndexFileReference(),
- componentFileRefrences.getBloomFilterFileReference(),
false);
+ LSMComponentFileReferences componentFileReferences =
fileManager.getTransactionFileReferenceForCommit();
+ AbstractDiskLSMComponent component = null;
+ if (componentFileReferences != null) {
+ component = createDiskComponent(componentFactory,
componentFileReferences.getInsertIndexFileReference(),
+ componentFileReferences.getBloomFilterFileReference(),
+ componentFileReferences.getStatisticsFileReference(),
false);
}
((ExternalIndexHarness)
lsmHarness).addTransactionComponents(component);
}
diff --git
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index 343f690..88a6eb4 100644
---
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -150,7 +150,8 @@
component = createDiskComponent(componentFactory,
lsmComonentFileReference.getInsertIndexFileReference(),
lsmComonentFileReference.getDeleteIndexFileReference(),
-
lsmComonentFileReference.getBloomFilterFileReference(), false);
+
lsmComonentFileReference.getBloomFilterFileReference(),
+
lsmComonentFileReference.getStatisticsFileReference(), false);
} catch (IndexException e) {
throw new HyracksDataException(e);
}
@@ -374,8 +375,8 @@
// which list to check against and we need to synchronize for this
boolean keepDeleteTuples = false;
if (version == 0) {
- keepDeleteTuples = mergingComponents.get(mergingComponents.size()
- 1) != diskComponents.get(diskComponents
- .size() - 1);
+ keepDeleteTuples = mergingComponents.get(mergingComponents.size()
- 1) != diskComponents
+ .get(diskComponents.size() - 1);
} else {
keepDeleteTuples = mergingComponents.get(mergingComponents.size()
- 1) != secondDiskComponents
.get(secondDiskComponents.size() - 1);
@@ -383,7 +384,8 @@
ioScheduler.scheduleOperation(new
LSMBTreeWithBuddyMergeOperation(accessor, mergingComponents, cursor,
relMergeFileRefs.getInsertIndexFileReference(),
relMergeFileRefs.getDeleteIndexFileReference(),
- relMergeFileRefs.getBloomFilterFileReference(), callback,
fileManager.getBaseDir(), keepDeleteTuples));
+ relMergeFileRefs.getBloomFilterFileReference(),
relMergeFileRefs.getStatisticsFileReference(), callback,
+ fileManager.getBaseDir(), keepDeleteTuples));
}
// This method creates the appropriate opContext for the targeted version
@@ -402,8 +404,8 @@
search(opCtx, cursor, btreeSearchPred);
LSMBTreeWithBuddyDiskComponent mergedComponent =
createDiskComponent(componentFactory,
- mergeOp.getBTreeMergeTarget(),
mergeOp.getBuddyBTreeMergeTarget(), mergeOp.getBloomFilterMergeTarget(),
- true);
+ mergeOp.getBTreeMergeTarget(),
mergeOp.getBuddyBTreeMergeTarget(), mergeOp.getBloomFilterTarget(),
+ mergeOp.getStatisticsMergeTarget(), true);
// In case we must keep the deleted-keys BuddyBTrees, then they must be
// merged *before* merging the b-trees so that
@@ -421,8 +423,8 @@
long numElements = 0L;
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
- numElements += ((LSMBTreeWithBuddyDiskComponent)
mergeOp.getMergingComponents().get(i))
- .getBloomFilter().getNumElements();
+ numElements += ((LSMBTreeWithBuddyDiskComponent)
mergeOp.getMergingComponents().get(i)).getBloomFilter()
+ .getNumElements();
}
int maxBucketsPerElement =
BloomCalculations.maxBucketsPerElement(numElements);
@@ -619,11 +621,10 @@
private LSMBTreeWithBuddyDiskComponent
createDiskComponent(ILSMComponentFactory factory,
FileReference insertFileRef, FileReference deleteFileRef,
FileReference bloomFilterFileRef,
- boolean createComponent) throws HyracksDataException,
IndexException {
+ FileReference statisticsFileRef, boolean createComponent) throws
HyracksDataException, IndexException {
// Create new instance.
- LSMBTreeWithBuddyDiskComponent component =
(LSMBTreeWithBuddyDiskComponent) factory
- .createLSMComponentInstance(new
LSMComponentFileReferences(insertFileRef, deleteFileRef,
- bloomFilterFileRef));
+ LSMBTreeWithBuddyDiskComponent component =
(LSMBTreeWithBuddyDiskComponent) factory.createLSMComponentInstance(
+ new LSMComponentFileReferences(insertFileRef, deleteFileRef,
bloomFilterFileRef, statisticsFileRef));
if (createComponent) {
component.getBTree().create();
component.getBuddyBTree().create();
@@ -798,7 +799,8 @@
protected ILSMComponent createBulkLoadTarget() throws
HyracksDataException, IndexException {
LSMComponentFileReferences componentFileRefs =
fileManager.getRelFlushFileReference();
return createDiskComponent(bulkComponentFactory,
componentFileRefs.getInsertIndexFileReference(),
- componentFileRefs.getDeleteIndexFileReference(),
componentFileRefs.getBloomFilterFileReference(), true);
+ componentFileRefs.getDeleteIndexFileReference(),
componentFileRefs.getBloomFilterFileReference(),
+ componentFileRefs.getStatisticsFileReference(), true);
}
// This method is used to create a target for a bulk modify operation. This
@@ -811,7 +813,8 @@
throw new HyracksDataException("Failed to create transaction
components", e);
}
return createDiskComponent(bulkComponentFactory,
componentFileRefs.getInsertIndexFileReference(),
- componentFileRefs.getDeleteIndexFileReference(),
componentFileRefs.getBloomFilterFileReference(), true);
+ componentFileRefs.getDeleteIndexFileReference(),
componentFileRefs.getBloomFilterFileReference(),
+ componentFileRefs.getStatisticsFileReference(), true);
}
@Override
@@ -849,12 +852,13 @@
@Override
public void commitTransaction() throws HyracksDataException,
IndexException {
- LSMComponentFileReferences componentFileRefrences =
fileManager.getTransactionFileReferenceForCommit();
+ LSMComponentFileReferences componentFileReferences =
fileManager.getTransactionFileReferenceForCommit();
LSMBTreeWithBuddyDiskComponent component = null;
- if (componentFileRefrences != null) {
- component = createDiskComponent(componentFactory,
componentFileRefrences.getInsertIndexFileReference(),
- componentFileRefrences.getDeleteIndexFileReference(),
- componentFileRefrences.getBloomFilterFileReference(),
false);
+ if (componentFileReferences != null) {
+ component = createDiskComponent(componentFactory,
componentFileReferences.getInsertIndexFileReference(),
+ componentFileReferences.getDeleteIndexFileReference(),
+ componentFileReferences.getBloomFilterFileReference(),
+ componentFileReferences.getStatisticsFileReference(),
false);
}
((ExternalIndexHarness)
lsmHarness).addTransactionComponents(component);
}
@@ -886,7 +890,7 @@
public boolean isPrimaryIndex() {
return false;
}
-
+
@Override
public Set<String> getLSMComponentPhysicalFiles(ILSMComponent
lsmComponent) {
Set<String> files = new HashSet<String>();
diff --git
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 663d2f7..d94072b 100644
---
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -74,6 +74,7 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import
org.apache.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractDiskLSMComponent;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
import
org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
import
org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
@@ -115,12 +116,13 @@
filterManager, filterFields, durable);
int i = 0;
for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
- LSMBTreeMemoryComponent mutableComponent = new
LSMBTreeMemoryComponent(new BTree(virtualBufferCache,
- virtualBufferCache.getFileMapProvider(), new
VirtualFreePageManager(
- virtualBufferCache.getNumPages()),
interiorFrameFactory, insertLeafFrameFactory,
- cmpFactories, fieldCount, new FileReference(new
File(fileManager.getBaseDir() + "_virtual_" + i))),
- virtualBufferCache, i == 0 ? true : false, filterFactory
== null ? null
- : filterFactory.createLSMComponentFilter());
+ LSMBTreeMemoryComponent mutableComponent = new
LSMBTreeMemoryComponent(
+ new BTree(virtualBufferCache,
virtualBufferCache.getFileMapProvider(),
+ new
VirtualFreePageManager(virtualBufferCache.getNumPages()), interiorFrameFactory,
+ insertLeafFrameFactory, cmpFactories, fieldCount,
+ new FileReference(new
File(fileManager.getBaseDir() + "_virtual_" + i))),
+ virtualBufferCache, i == 0 ? true : false,
+ filterFactory == null ? null :
filterFactory.createLSMComponentFilter());
memoryComponents.add(mutableComponent);
++i;
}
@@ -185,11 +187,12 @@
throw new HyracksDataException(e);
}
for (LSMComponentFileReferences lsmComonentFileReference :
validFileReferences) {
- LSMBTreeDiskComponent component;
+ AbstractDiskLSMComponent component;
try {
component = createDiskComponent(componentFactory,
lsmComonentFileReference.getInsertIndexFileReference(),
-
lsmComonentFileReference.getBloomFilterFileReference(), false);
+ lsmComonentFileReference.getBloomFilterFileReference(),
+ lsmComonentFileReference.getStatisticsFileReference(),
false);
} catch (IndexException e) {
throw new HyracksDataException(e);
}
@@ -371,8 +374,8 @@
}
if (ctx.filterTuple != null) {
ctx.filterTuple.reset(tuple);
-
memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter()
- .update(ctx.filterTuple, ctx.filterCmp);
+
memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter().update(ctx.filterTuple,
+ ctx.filterCmp);
}
}
@@ -450,9 +453,9 @@
opCtx.setOperation(IndexOperation.FLUSH);
opCtx.getComponentHolder().add(flushingComponent);
ILSMIndexAccessorInternal flushAccessor = new
LSMBTreeAccessor(lsmHarness, opCtx);
- ioScheduler.scheduleOperation(new
LSMBTreeFlushOperation(flushAccessor, flushingComponent, componentFileRefs
- .getInsertIndexFileReference(),
componentFileRefs.getBloomFilterFileReference(), callback, fileManager
- .getBaseDir()));
+ ioScheduler.scheduleOperation(new
LSMBTreeFlushOperation(flushAccessor, flushingComponent,
+ componentFileRefs.getInsertIndexFileReference(),
componentFileRefs.getBloomFilterFileReference(),
+ componentFileRefs.getStatisticsFileReference(), callback,
fileManager.getBaseDir()));
}
@Override
@@ -481,10 +484,10 @@
bloomFilterFalsePositiveRate);
LSMBTreeDiskComponent component =
createDiskComponent(componentFactory, flushOp.getBTreeFlushTarget(),
- flushOp.getBloomFilterFlushTarget(), true);
+ flushOp.getBloomFilterTarget(), flushOp.getStatisticsTarget(),
true);
IIndexBulkLoader bulkLoader =
component.getBTree().createBulkLoader(1.0f, false, numElements, false);
- IIndexBulkLoader builder =
component.getBloomFilter().createBuilder(numElements,
- bloomFilterSpec.getNumHashes(),
bloomFilterSpec.getNumBucketsPerElements());
+ IIndexBulkLoader builder =
component.getBloomFilter().createBuilder(numElements,
bloomFilterSpec.getNumHashes(),
+ bloomFilterSpec.getNumBucketsPerElements());
IIndexCursor scanCursor = accessor.createSearchCursor(false);
accessor.search(scanCursor, nullPred);
@@ -517,8 +520,8 @@
opCtx.setOperation(IndexOperation.MERGE);
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
boolean returnDeletedTuples = false;
- if (ctx.getComponentHolder().get(ctx.getComponentHolder().size() - 1)
!= diskComponents.get(diskComponents
- .size() - 1)) {
+ if (ctx.getComponentHolder().get(ctx.getComponentHolder().size() - 1)
!= diskComponents
+ .get(diskComponents.size() - 1)) {
returnDeletedTuples = true;
}
ITreeIndexCursor cursor = new LSMBTreeRangeSearchCursor(opCtx,
returnDeletedTuples);
@@ -526,12 +529,12 @@
BTree lastBTree = ((LSMBTreeDiskComponent)
mergingComponents.get(mergingComponents.size() - 1)).getBTree();
FileReference firstFile =
diskFileMapProvider.lookupFileName(firstBTree.getFileId());
FileReference lastFile =
diskFileMapProvider.lookupFileName(lastBTree.getFileId());
- LSMComponentFileReferences relMergeFileRefs =
fileManager.getRelMergeFileReference(firstFile.getFile()
- .getName(), lastFile.getFile().getName());
+ LSMComponentFileReferences relMergeFileRefs = fileManager
+ .getRelMergeFileReference(firstFile.getFile().getName(),
lastFile.getFile().getName());
ILSMIndexAccessorInternal accessor = new LSMBTreeAccessor(lsmHarness,
opCtx);
- ioScheduler.scheduleOperation(new LSMBTreeMergeOperation(accessor,
mergingComponents, cursor, relMergeFileRefs
- .getInsertIndexFileReference(),
relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager
- .getBaseDir()));
+ ioScheduler.scheduleOperation(new LSMBTreeMergeOperation(accessor,
mergingComponents, cursor,
+ relMergeFileRefs.getInsertIndexFileReference(),
relMergeFileRefs.getBloomFilterFileReference(),
+ relMergeFileRefs.getStatisticsFileReference(), callback,
fileManager.getBaseDir()));
}
@Override
@@ -546,14 +549,14 @@
long numElements = 0L;
for (int i = 0; i < mergedComponents.size(); ++i) {
- numElements += ((LSMBTreeDiskComponent)
mergedComponents.get(i)).getBloomFilter().getNumElements();
+ numElements += ((AbstractDiskLSMComponent)
mergedComponents.get(i)).getBloomFilter().getNumElements();
}
int maxBucketsPerElement =
BloomCalculations.maxBucketsPerElement(numElements);
BloomFilterSpecification bloomFilterSpec =
BloomCalculations.computeBloomSpec(maxBucketsPerElement,
bloomFilterFalsePositiveRate);
LSMBTreeDiskComponent mergedComponent =
createDiskComponent(componentFactory, mergeOp.getBTreeMergeTarget(),
- mergeOp.getBloomFilterMergeTarget(), true);
+ mergeOp.getBloomFilterTarget(),
mergeOp.getStatisticsMergeTarget(), true);
IIndexBulkLoader bulkLoader =
mergedComponent.getBTree().createBulkLoader(1.0f, false, numElements, false);
IIndexBulkLoader builder =
mergedComponent.getBloomFilter().createBuilder(numElements,
@@ -585,11 +588,11 @@
}
protected LSMBTreeDiskComponent
createDiskComponent(LSMBTreeDiskComponentFactory factory,
- FileReference btreeFileRef, FileReference bloomFilterFileRef,
boolean createComponent)
- throws HyracksDataException, IndexException {
+ FileReference btreeFileRef, FileReference bloomFilterFileRef,
FileReference statisticsFileReference,
+ boolean createComponent) throws HyracksDataException,
IndexException {
// Create new BTree instance.
- LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) factory
- .createLSMComponentInstance(new
LSMComponentFileReferences(btreeFileRef, null, bloomFilterFileRef));
+ LSMBTreeDiskComponent component = (LSMBTreeDiskComponent)
factory.createLSMComponentInstance(
+ new LSMComponentFileReferences(btreeFileRef, null,
bloomFilterFileRef, statisticsFileReference));
if (createComponent) {
component.getBTree().create();
component.getBloomFilter().create();
@@ -616,7 +619,7 @@
protected ILSMComponent createBulkLoadTarget() throws
HyracksDataException, IndexException {
LSMComponentFileReferences componentFileRefs =
fileManager.getRelFlushFileReference();
return createDiskComponent(bulkLoadComponentFactory,
componentFileRefs.getInsertIndexFileReference(),
- componentFileRefs.getBloomFilterFileReference(), true);
+ componentFileRefs.getBloomFilterFileReference(),
componentFileRefs.getStatisticsFileReference(), true);
}
@Override
@@ -645,8 +648,8 @@
public final PermutingTupleReference filterTuple;
public final MultiComparator filterCmp;
- public LSMBTreeBulkLoader(float fillFactor, boolean verifyInput, long
numElementsHint, boolean checkIfEmptyIndex)
- throws TreeIndexException, HyracksDataException {
+ public LSMBTreeBulkLoader(float fillFactor, boolean verifyInput, long
numElementsHint,
+ boolean checkIfEmptyIndex) throws TreeIndexException,
HyracksDataException {
if (checkIfEmptyIndex && !isEmptyIndex()) {
throw new TreeIndexException("Cannot load an index that is not
empty");
}
@@ -661,7 +664,7 @@
int maxBucketsPerElement =
BloomCalculations.maxBucketsPerElement(numElementsHint);
BloomFilterSpecification bloomFilterSpec =
BloomCalculations.computeBloomSpec(maxBucketsPerElement,
bloomFilterFalsePositiveRate);
- builder = ((LSMBTreeDiskComponent)
component).getBloomFilter().createBuilder(numElementsHint,
+ builder = ((AbstractDiskLSMComponent)
component).getBloomFilter().createBuilder(numElementsHint,
bloomFilterSpec.getNumHashes(),
bloomFilterSpec.getNumBucketsPerElements());
if (filterFields != null) {
@@ -712,8 +715,8 @@
}
((LSMBTreeDiskComponent) component).getBTree().deactivate();
((LSMBTreeDiskComponent) component).getBTree().destroy();
- ((LSMBTreeDiskComponent)
component).getBloomFilter().deactivate();
- ((LSMBTreeDiskComponent) component).getBloomFilter().destroy();
+ ((AbstractDiskLSMComponent)
component).getBloomFilter().deactivate();
+ ((AbstractDiskLSMComponent)
component).getBloomFilter().destroy();
}
}
@@ -863,7 +866,7 @@
files.add(component.getBTree().getFileReference().toString());
files.add(component.getBloomFilter().getFileReference().toString());
-
+
return files;
}
}
diff --git
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
index f553368..0be3213 100644
---
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
+++
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
@@ -26,28 +26,21 @@
public class LSMBTreeDiskComponent extends AbstractDiskLSMComponent {
private final BTree btree;
- private final BloomFilter bloomFilter;
public LSMBTreeDiskComponent(BTree btree, BloomFilter bloomFilter,
ILSMComponentFilter filter) {
- super(filter);
+ super(bloomFilter, filter);
this.btree = btree;
- this.bloomFilter = bloomFilter;
}
@Override
public void destroy() throws HyracksDataException {
btree.deactivate();
btree.destroy();
- bloomFilter.deactivate();
- bloomFilter.destroy();
+ super.destroy();
}
public BTree getBTree() {
return btree;
- }
-
- public BloomFilter getBloomFilter() {
- return bloomFilter;
}
@Override
diff --git
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
index 2be4e4a..2eaf792 100644
---
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
+++
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
@@ -56,7 +56,8 @@
String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a flush
return new LSMComponentFileReferences(createFlushFile(baseName +
SPLIT_STRING + BTREE_STRING), null,
- createFlushFile(baseName + SPLIT_STRING +
BLOOM_FILTER_STRING));
+ createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING),
+ createFlushFile(baseName + SPLIT_STRING + STATISTICS_STRING));
}
@Override
@@ -68,7 +69,8 @@
String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING +
lastTimestampRange[1];
// Get the range of timestamps by taking the earliest and the latest
timestamps
return new LSMComponentFileReferences(createMergeFile(baseName +
SPLIT_STRING + BTREE_STRING), null,
- createMergeFile(baseName + SPLIT_STRING +
BLOOM_FILTER_STRING));
+ createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING),
+ createFlushFile(baseName + SPLIT_STRING + STATISTICS_STRING));
}
private static FilenameFilter btreeFilter = new FilenameFilter() {
@@ -96,7 +98,8 @@
int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
}
- validateFiles(btreeFilesSet, allBloomFilterFiles,
getCompoundFilter(transactionFilter, bloomFilterFilter), null);
+ validateFiles(btreeFilesSet, allBloomFilterFiles,
getCompoundFilter(transactionFilter, bloomFilterFilter),
+ null);
// Sanity check.
if (allBTreeFiles.size() != allBloomFilterFiles.size()) {
@@ -110,8 +113,8 @@
}
if (allBTreeFiles.size() == 1 && allBloomFilterFiles.size() == 1) {
- validFiles.add(new
LSMComponentFileReferences(allBTreeFiles.get(0).fileRef, null,
allBloomFilterFiles
- .get(0).fileRef));
+ validFiles.add(new
LSMComponentFileReferences(allBTreeFiles.get(0).fileRef, null,
+ allBloomFilterFiles.get(0).fileRef, null));
return validFiles;
}
@@ -148,7 +151,8 @@
invalidBloomFilterFile.delete();
} else {
// This scenario should not be possible.
- throw new HyracksDataException("Found LSM files with
overlapping but not contained timetamp intervals.");
+ throw new HyracksDataException(
+ "Found LSM files with overlapping but not contained
timetamp intervals.");
}
}
@@ -163,7 +167,7 @@
ComparableFileName cmpBTreeFileName = btreeFileIter.next();
ComparableFileName cmpBloomFilterFileName =
bloomFilterFileIter.next();
validFiles.add(new
LSMComponentFileReferences(cmpBTreeFileName.fileRef, null,
- cmpBloomFilterFileName.fileRef));
+ cmpBloomFilterFileName.fileRef, null));
}
return validFiles;
@@ -178,7 +182,7 @@
String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a
transaction
return new LSMComponentFileReferences(createFlushFile(baseName +
SPLIT_STRING + BTREE_STRING), null,
- createFlushFile(baseName + SPLIT_STRING +
BLOOM_FILTER_STRING));
+ createFlushFile(baseName + SPLIT_STRING +
BLOOM_FILTER_STRING), null);
}
@Override
@@ -220,6 +224,6 @@
FileReference bTreeFileRef = new FileReference(bTreeFile);
FileReference bloomFilterFileRef = new FileReference(bloomFilterFile);
- return new LSMComponentFileReferences(bTreeFileRef, null,
bloomFilterFileRef);
+ return new LSMComponentFileReferences(bTreeFileRef, null,
bloomFilterFileRef, null);
}
}
diff --git
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java
index c56293f..9035c07 100644
---
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java
+++
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java
@@ -18,87 +18,42 @@
*/
package org.apache.hyracks.storage.am.lsm.btree.impls;
-import java.util.Collections;
-import java.util.HashSet;
import java.util.Set;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IODeviceHandle;
-import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMFlushOperation;
-public class LSMBTreeFlushOperation implements ILSMIOOperation,
Comparable<LSMBTreeFlushOperation> {
+public class LSMBTreeFlushOperation extends AbstractLSMFlushOperation
implements Comparable<LSMBTreeFlushOperation> {
- private final ILSMIndexAccessorInternal accessor;
- private final ILSMComponent flushingComponent;
private final FileReference btreeFlushTarget;
- private final FileReference bloomFilterFlushTarget;
- private final ILSMIOOperationCallback callback;
- private final String indexIdentifier;
+ private final FileReference statisticsFlushTarget;
public LSMBTreeFlushOperation(ILSMIndexAccessorInternal accessor,
ILSMComponent flushingComponent,
- FileReference btreeFlushTarget, FileReference
bloomFilterFlushTarget, ILSMIOOperationCallback callback,
- String indexIdentifier) {
- this.accessor = accessor;
- this.flushingComponent = flushingComponent;
+ FileReference btreeFlushTarget, FileReference
bloomFilterFlushTarget, FileReference statisticsFlushTarget,
+ ILSMIOOperationCallback callback, String indexIdentifier) {
+ super(accessor, flushingComponent, bloomFilterFlushTarget, callback,
indexIdentifier);
this.btreeFlushTarget = btreeFlushTarget;
- this.bloomFilterFlushTarget = bloomFilterFlushTarget;
- this.callback = callback;
- this.indexIdentifier = indexIdentifier;
- }
-
- @Override
- public Set<IODeviceHandle> getReadDevices() {
- return Collections.emptySet();
+ this.statisticsFlushTarget = statisticsFlushTarget;
}
@Override
public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ Set<IODeviceHandle> devs = super.getWriteDevices();
devs.add(btreeFlushTarget.getDeviceHandle());
- devs.add(bloomFilterFlushTarget.getDeviceHandle());
+ devs.add(statisticsFlushTarget.getDeviceHandle());
return devs;
- }
-
- @Override
- public Boolean call() throws HyracksDataException, IndexException {
- accessor.flush(this);
- return true;
- }
-
- @Override
- public ILSMIOOperationCallback getCallback() {
- return callback;
}
public FileReference getBTreeFlushTarget() {
return btreeFlushTarget;
}
- public FileReference getBloomFilterFlushTarget() {
- return bloomFilterFlushTarget;
- }
-
- public ILSMIndexAccessorInternal getAccessor() {
- return accessor;
- }
-
- public ILSMComponent getFlushingComponent() {
- return flushingComponent;
- }
-
- @Override
- public String getIndexUniqueIdentifier() {
- return indexIdentifier;
- }
-
- @Override
- public LSMIOOpertionType getIOOpertionType() {
- return LSMIOOpertionType.FLUSH;
+ public FileReference getStatisticsTarget() {
+ return statisticsFlushTarget;
}
@Override
diff --git
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java
index f69a77f..6b69021 100644
---
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java
+++
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java
@@ -19,95 +19,54 @@
package org.apache.hyracks.storage.am.lsm.btree.impls;
-import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
-import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMMergeOperation;
-public class LSMBTreeMergeOperation implements ILSMIOOperation {
+public class LSMBTreeMergeOperation extends AbstractLSMMergeOperation
implements ILSMIOOperation {
- private final ILSMIndexAccessorInternal accessor;
- private final List<ILSMComponent> mergingComponents;
- private final ITreeIndexCursor cursor;
private final FileReference btreeMergeTarget;
- private final FileReference bloomFilterMergeTarget;
- private final ILSMIOOperationCallback callback;
- private final String indexIdentifier;
+ private final FileReference statisticsMergeTarget;
public LSMBTreeMergeOperation(ILSMIndexAccessorInternal accessor,
List<ILSMComponent> mergingComponents,
ITreeIndexCursor cursor, FileReference btreeMergeTarget,
FileReference bloomFilterMergeTarget,
- ILSMIOOperationCallback callback, String indexIdentifier) {
- this.accessor = accessor;
- this.mergingComponents = mergingComponents;
- this.cursor = cursor;
+ FileReference statisticsMergeTarget, ILSMIOOperationCallback
callback, String indexIdentifier) {
+ super(accessor, mergingComponents, cursor, bloomFilterMergeTarget,
callback, indexIdentifier);
this.btreeMergeTarget = btreeMergeTarget;
- this.bloomFilterMergeTarget = bloomFilterMergeTarget;
- this.callback = callback;
- this.indexIdentifier = indexIdentifier;
+ this.statisticsMergeTarget = statisticsMergeTarget;
}
@Override
public Set<IODeviceHandle> getReadDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ Set<IODeviceHandle> devs = super.getReadDevices();
for (ILSMComponent o : mergingComponents) {
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) o;
devs.add(component.getBTree().getFileReference().getDeviceHandle());
-
devs.add(component.getBloomFilter().getFileReference().getDeviceHandle());
}
return devs;
}
@Override
public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ Set<IODeviceHandle> devs = super.getWriteDevices();
devs.add(btreeMergeTarget.getDeviceHandle());
- devs.add(bloomFilterMergeTarget.getDeviceHandle());
+ devs.add(statisticsMergeTarget.getDeviceHandle());
return devs;
- }
-
- @Override
- public Boolean call() throws HyracksDataException, IndexException {
- accessor.merge(this);
- return true;
- }
-
- @Override
- public ILSMIOOperationCallback getCallback() {
- return callback;
}
public FileReference getBTreeMergeTarget() {
return btreeMergeTarget;
}
- public FileReference getBloomFilterMergeTarget() {
- return bloomFilterMergeTarget;
- }
-
- public ITreeIndexCursor getCursor() {
- return cursor;
- }
-
- public List<ILSMComponent> getMergingComponents() {
- return mergingComponents;
- }
-
- @Override
- public String getIndexUniqueIdentifier() {
- return indexIdentifier;
- }
-
- @Override
- public LSMIOOpertionType getIOOpertionType() {
- return LSMIOOpertionType.MERGE;
+ public FileReference getStatisticsMergeTarget() {
+ return statisticsMergeTarget;
}
}
diff --git
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
index 8d442d6..41cabea 100644
---
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
+++
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
@@ -40,6 +40,7 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractDiskLSMComponent;
import
org.apache.hyracks.storage.am.lsm.common.impls.BloomFilterAwareBTreePointSearchCursor;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
@@ -166,7 +167,7 @@
btree = (BTree) ((LSMBTreeMemoryComponent)
component).getBTree();
} else {
rangeCursors[i] = new
BloomFilterAwareBTreePointSearchCursor(leafFrame, false,
- ((LSMBTreeDiskComponent) component).getBloomFilter());
+ ((AbstractDiskLSMComponent)
component).getBloomFilter());
btree = (BTree) ((LSMBTreeDiskComponent) component).getBTree();
}
btreeAccessors[i] =
btree.createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
diff --git
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java
index 3845740..a9ea547 100644
---
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java
+++
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java
@@ -27,12 +27,11 @@
private final BTree btree;
private final BTree buddyBtree;
- private final BloomFilter bloomFilter;
public LSMBTreeWithBuddyDiskComponent(BTree btree, BTree buddyBtree,
BloomFilter bloomFilter) {
+ super(bloomFilter, null);
this.btree = btree;
this.buddyBtree = buddyBtree;
- this.bloomFilter = bloomFilter;
}
@Override
@@ -41,8 +40,7 @@
btree.destroy();
buddyBtree.deactivate();
buddyBtree.destroy();
- bloomFilter.deactivate();
- bloomFilter.destroy();
+ super.destroy();
}
public BTree getBTree() {
@@ -53,15 +51,11 @@
return buddyBtree;
}
- public BloomFilter getBloomFilter() {
- return bloomFilter;
- }
-
@Override
public long getComponentSize() {
- long size = btree.getFileReference().getFile().length();
+ long size = super.getComponentSize();
+ size = btree.getFileReference().getFile().length();
size += buddyBtree.getFileReference().getFile().length();
- size += bloomFilter.getFileReference().getFile().length();
return size;
}
diff --git
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
index ed39630..5d45f08 100644
---
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
+++
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
@@ -73,8 +73,9 @@
String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a flush
return new LSMComponentFileReferences(createFlushFile(baseName +
SPLIT_STRING + BTREE_STRING),
- createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING),
createFlushFile(baseName + SPLIT_STRING
- + BLOOM_FILTER_STRING));
+ createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING),
+ createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING),
+ createFlushFile(baseName + SPLIT_STRING + STATISTICS_STRING));
}
@Override
@@ -87,8 +88,9 @@
// Get the range of timestamps by taking the earliest and the latest
// timestamps
return new LSMComponentFileReferences(createMergeFile(baseName +
SPLIT_STRING + BTREE_STRING),
- createMergeFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING),
createMergeFile(baseName + SPLIT_STRING
- + BLOOM_FILTER_STRING));
+ createMergeFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING),
+ createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING),
+ createFlushFile(baseName + SPLIT_STRING + STATISTICS_STRING));
}
@Override
@@ -127,7 +129,7 @@
if (allBTreeFiles.size() == 1 && allBuddyBTreeFiles.size() == 1 &&
allBloomFilterFiles.size() == 1) {
validFiles.add(new
LSMComponentFileReferences(allBTreeFiles.get(0).fileRef,
- allBuddyBTreeFiles.get(0).fileRef,
allBloomFilterFiles.get(0).fileRef));
+ allBuddyBTreeFiles.get(0).fileRef,
allBloomFilterFiles.get(0).fileRef, null));
return validFiles;
}
@@ -177,7 +179,8 @@
invalidBloomFilterFile.delete();
} else {
// This scenario should not be possible.
- throw new HyracksDataException("Found LSM files with
overlapping but not contained timetamp intervals.");
+ throw new HyracksDataException(
+ "Found LSM files with overlapping but not contained
timetamp intervals.");
}
}
@@ -195,7 +198,7 @@
ComparableFileName cmpBuddyBTreeFileName =
buddyBtreeFileIter.next();
ComparableFileName cmpBloomFilterFileName =
bloomFilterFileIter.next();
validFiles.add(new
LSMComponentFileReferences(cmpBTreeFileName.fileRef,
cmpBuddyBTreeFileName.fileRef,
- cmpBloomFilterFileName.fileRef));
+ cmpBloomFilterFileName.fileRef, null));
}
return validFiles;
@@ -209,8 +212,8 @@
String baseName = baseDir + ts + SPLIT_STRING + ts;
return new LSMComponentFileReferences(createFlushFile(baseName +
SPLIT_STRING + BTREE_STRING),
- createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING),
createFlushFile(baseName + SPLIT_STRING
- + BLOOM_FILTER_STRING));
+ createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING),
+ createFlushFile(baseName + SPLIT_STRING +
BLOOM_FILTER_STRING), null);
}
@Override
@@ -229,8 +232,8 @@
// get the actual transaction files
files = dir.list(transactionFilter);
if (files.length < 3) {
- throw new HyracksDataException("LSM Btree with buddy
transaction has less than 3 files :"
- + files.length);
+ throw new HyracksDataException(
+ "LSM Btree with buddy transaction has less than 3
files :" + files.length);
}
try {
Files.delete(Paths.get(txnFileName));
@@ -255,7 +258,7 @@
FileReference bTreeFileRef = new FileReference(bTreeFile);
FileReference buddyBTreeFileRef = new FileReference(buddyBTreeFile);
FileReference bloomFilterFileRef = new FileReference(bloomFilterFile);
- return new LSMComponentFileReferences(bTreeFileRef, buddyBTreeFileRef,
bloomFilterFileRef);
+ return new LSMComponentFileReferences(bTreeFileRef, buddyBTreeFileRef,
bloomFilterFileRef, null);
}
}
diff --git
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java
index 61f68fb..c5fcc23 100644
---
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java
+++
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java
@@ -22,115 +22,55 @@
import java.util.List;
import java.util.Set;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
-import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
-public class LSMBTreeWithBuddyMergeOperation implements ILSMIOOperation {
+public class LSMBTreeWithBuddyMergeOperation extends LSMBTreeMergeOperation {
- private final ILSMIndexAccessorInternal accessor;
- private final List<ILSMComponent> mergingComponents;
- private final ITreeIndexCursor cursor;
- private final FileReference btreeMergeTarget;
- private final FileReference buddyBtreeMergeTarget;
- private final FileReference bloomFilterMergeTarget;
- private final ILSMIOOperationCallback callback;
- private final String indexIdentifier;
- private final boolean keepDeletedTuples;
+ private final FileReference buddyBtreeMergeTarget;
+ private final boolean keepDeletedTuples;
- public LSMBTreeWithBuddyMergeOperation(ILSMIndexAccessorInternal
accessor,
- List<ILSMComponent> mergingComponents, ITreeIndexCursor
cursor,
- FileReference btreeMergeTarget,
- FileReference buddyBtreeMergeTarget,
- FileReference bloomFilterMergeTarget,
- ILSMIOOperationCallback callback, String
indexIdentifier, boolean keepDeletedTuples) {
- this.accessor = accessor;
- this.mergingComponents = mergingComponents;
- this.cursor = cursor;
- this.btreeMergeTarget = btreeMergeTarget;
- this.buddyBtreeMergeTarget = buddyBtreeMergeTarget;
- this.bloomFilterMergeTarget = bloomFilterMergeTarget;
- this.callback = callback;
- this.indexIdentifier = indexIdentifier;
- this.keepDeletedTuples = keepDeletedTuples;
- }
+ public LSMBTreeWithBuddyMergeOperation(ILSMIndexAccessorInternal accessor,
List<ILSMComponent> mergingComponents,
+ ITreeIndexCursor cursor, FileReference btreeMergeTarget,
FileReference buddyBtreeMergeTarget,
+ FileReference bloomFilterMergeTarget, FileReference
statisticsMergeTarget, ILSMIOOperationCallback callback,
+ String indexIdentifier, boolean keepDeletedTuples) {
+ super(accessor, mergingComponents, cursor, btreeMergeTarget,
bloomFilterMergeTarget, statisticsMergeTarget,
+ callback, indexIdentifier);
+ this.buddyBtreeMergeTarget = buddyBtreeMergeTarget;
+ this.keepDeletedTuples = keepDeletedTuples;
+ }
- @Override
- public Set<IODeviceHandle> getReadDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
- for (ILSMComponent o : mergingComponents) {
- LSMBTreeWithBuddyDiskComponent component =
(LSMBTreeWithBuddyDiskComponent) o;
-
devs.add(component.getBTree().getFileReference().getDeviceHandle());
+ @Override
+ public Set<IODeviceHandle> getReadDevices() {
+ Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ for (ILSMComponent o : mergingComponents) {
+ LSMBTreeWithBuddyDiskComponent component =
(LSMBTreeWithBuddyDiskComponent) o;
+
devs.add(component.getBTree().getFileReference().getDeviceHandle());
- devs.add(component.getBuddyBTree().getFileReference()
- .getDeviceHandle());
- devs.add(component.getBloomFilter().getFileReference()
- .getDeviceHandle());
+
devs.add(component.getBuddyBTree().getFileReference().getDeviceHandle());
+
devs.add(component.getBloomFilter().getFileReference().getDeviceHandle());
- }
- return devs;
- }
+ }
+ return devs;
+ }
- @Override
- public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
- devs.add(btreeMergeTarget.getDeviceHandle());
+ @Override
+ public Set<IODeviceHandle> getWriteDevices() {
+ Set<IODeviceHandle> devs = super.getWriteDevices();
+ devs.add(buddyBtreeMergeTarget.getDeviceHandle());
+ return devs;
+ }
- devs.add(buddyBtreeMergeTarget.getDeviceHandle());
- devs.add(bloomFilterMergeTarget.getDeviceHandle());
+ public FileReference getBuddyBTreeMergeTarget() {
+ return buddyBtreeMergeTarget;
+ }
- return devs;
- }
-
- @Override
- public Boolean call() throws HyracksDataException, IndexException {
- accessor.merge(this);
- return true;
- }
-
- @Override
- public ILSMIOOperationCallback getCallback() {
- return callback;
- }
-
- @Override
- public String getIndexUniqueIdentifier() {
- return indexIdentifier;
- }
-
- @Override
- public LSMIOOpertionType getIOOpertionType() {
- return LSMIOOpertionType.MERGE;
- }
-
- public FileReference getBTreeMergeTarget() {
- return btreeMergeTarget;
- }
-
- public FileReference getBuddyBTreeMergeTarget() {
- return buddyBtreeMergeTarget;
- }
-
- public FileReference getBloomFilterMergeTarget() {
- return bloomFilterMergeTarget;
- }
-
- public ITreeIndexCursor getCursor() {
- return cursor;
- }
-
- public List<ILSMComponent> getMergingComponents() {
- return mergingComponents;
- }
-
- public boolean isKeepDeletedTuples() {
- return keepDeletedTuples;
- }
+ public boolean isKeepDeletedTuples() {
+ return keepDeletedTuples;
+ }
}
diff --git
a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractDiskLSMComponent.java
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractDiskLSMComponent.java
index 54ec029..952440c 100644
---
a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractDiskLSMComponent.java
+++
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractDiskLSMComponent.java
@@ -19,18 +19,22 @@
package org.apache.hyracks.storage.am.lsm.common.impls;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
public abstract class AbstractDiskLSMComponent extends AbstractLSMComponent {
- public AbstractDiskLSMComponent(ILSMComponentFilter filter) {
+ protected final BloomFilter bloomFilter;
+
+ public AbstractDiskLSMComponent(BloomFilter bloomFilter,
ILSMComponentFilter filter) {
super(filter);
+ this.bloomFilter = bloomFilter;
state = ComponentState.READABLE_UNWRITABLE;
}
public AbstractDiskLSMComponent() {
- this(null);
+ this(null, null);
}
@Override
@@ -100,9 +104,18 @@
return state;
}
- protected abstract void destroy() throws HyracksDataException;
+ public BloomFilter getBloomFilter() {
+ return bloomFilter;
+ }
- public abstract long getComponentSize();
+ protected void destroy() throws HyracksDataException {
+ bloomFilter.deactivate();
+ bloomFilter.destroy();
+ }
+
+ public long getComponentSize() {
+ return bloomFilter.getFileReference().getFile().length();
+ }
public abstract int getFileReferenceCount();
diff --git
a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
index 3ad1396..39ff3d0 100644
---
a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
+++
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
@@ -49,6 +49,7 @@
public static final String SPLIT_STRING = "_";
protected static final String BLOOM_FILTER_STRING = "f";
protected static final String TRANSACTION_PREFIX = ".T";
+ protected static final String STATISTICS_STRING = "s";
protected final IFileMapProvider fileMapProvider;
@@ -117,8 +118,8 @@
}
protected void validateFiles(HashSet<String> groundTruth,
ArrayList<ComparableFileName> validFiles,
- FilenameFilter filter, TreeIndexFactory<? extends ITreeIndex>
treeFactory) throws HyracksDataException,
- IndexException {
+ FilenameFilter filter, TreeIndexFactory<? extends ITreeIndex>
treeFactory)
+ throws HyracksDataException, IndexException {
ArrayList<ComparableFileName> tmpAllInvListsFiles = new
ArrayList<ComparableFileName>();
cleanupAndGetValidFilesInternal(filter, treeFactory,
tmpAllInvListsFiles);
for (ComparableFileName cmpFileName : tmpAllInvListsFiles) {
@@ -173,7 +174,7 @@
public LSMComponentFileReferences getRelFlushFileReference() {
String ts = getCurrentTimestamp();
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseDir + ts +
SPLIT_STRING + ts), null, null);
+ return new LSMComponentFileReferences(createFlushFile(baseDir + ts +
SPLIT_STRING + ts), null, null, null);
}
@Override
@@ -182,8 +183,9 @@
String[] firstTimestampRange = firstFileName.split(SPLIT_STRING);
String[] lastTimestampRange = lastFileName.split(SPLIT_STRING);
// Get the range of timestamps by taking the earliest and the latest
timestamps
- return new LSMComponentFileReferences(createMergeFile(baseDir +
firstTimestampRange[0] + SPLIT_STRING
- + lastTimestampRange[1]), null, null);
+ return new LSMComponentFileReferences(
+ createMergeFile(baseDir + firstTimestampRange[0] +
SPLIT_STRING + lastTimestampRange[1]), null, null,
+ null);
}
@Override
@@ -203,7 +205,7 @@
}
if (allFiles.size() == 1) {
- validFiles.add(new
LSMComponentFileReferences(allFiles.get(0).fileRef, null, null));
+ validFiles.add(new
LSMComponentFileReferences(allFiles.get(0).fileRef, null, null, null));
return validFiles;
}
@@ -235,7 +237,7 @@
// Sort valid files in reverse lexicographical order, such that newer
files come first.
Collections.sort(validComparableFiles, recencyCmp);
for (ComparableFileName cmpFileName : validComparableFiles) {
- validFiles.add(new LSMComponentFileReferences(cmpFileName.fileRef,
null, null));
+ validFiles.add(new LSMComponentFileReferences(cmpFileName.fileRef,
null, null, null));
}
return validFiles;
@@ -375,8 +377,8 @@
};
protected static FilenameFilter createTransactionFilter(String
transactionFileName, final boolean inclusive) {
- final String timeStamp =
transactionFileName.substring(transactionFileName.indexOf(TRANSACTION_PREFIX)
- + TRANSACTION_PREFIX.length());
+ final String timeStamp = transactionFileName
+ .substring(transactionFileName.indexOf(TRANSACTION_PREFIX) +
TRANSACTION_PREFIX.length());
return new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
diff --git
a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java
index 95fddfc..baebf65 100644
---
a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java
+++
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java
@@ -30,12 +30,15 @@
// This FileReference for the bloom filter (if any).
private final FileReference bloomFilterFileReference;
+ // This FileReference for the component statistics (if any).
+ private final FileReference statisticsFileReference;
public LSMComponentFileReferences(FileReference insertIndexFileReference,
FileReference deleteIndexFileReference,
- FileReference bloomFilterFileReference) {
+ FileReference bloomFilterFileReference, FileReference
statisticsFileReference) {
this.insertIndexFileReference = insertIndexFileReference;
this.deleteIndexFileReference = deleteIndexFileReference;
this.bloomFilterFileReference = bloomFilterFileReference;
+ this.statisticsFileReference = statisticsFileReference;
}
public FileReference getInsertIndexFileReference() {
@@ -49,4 +52,8 @@
public FileReference getBloomFilterFileReference() {
return bloomFilterFileReference;
}
+
+ public FileReference getStatisticsFileReference() {
+ return statisticsFileReference;
+ }
}
diff --git
a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 28b96db..ad41c4e 100644
---
a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++
b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -46,6 +46,7 @@
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.api.IVirtualFreePageManager;
import org.apache.hyracks.storage.am.common.api.IndexException;
import
org.apache.hyracks.storage.am.common.exceptions.TreeIndexDuplicateKeyException;
@@ -109,14 +110,13 @@
OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory
deletedKeysBTreeFactory,
BloomFilterFactory bloomFilterFactory, ILSMComponentFilterFactory
filterFactory,
ILSMComponentFilterFrameFactory filterFrameFactory,
LSMComponentFilterManager filterManager,
- double bloomFilterFalsePositiveRate, ILSMIndexFileManager
fileManager,
- IFileMapProvider diskFileMapProvider, ITypeTraits[]
invListTypeTraits,
- IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[]
tokenTypeTraits,
- IBinaryComparatorFactory[] tokenCmpFactories,
IBinaryTokenizerFactory tokenizerFactory,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields,
int[] filterFields,
- int[] filterFieldsForNonBulkLoadOps, int[]
invertedIndexFieldsForNonBulkLoadOps, boolean durable)
- throws IndexException {
+ double bloomFilterFalsePositiveRate, ILSMIndexFileManager
fileManager, IFileMapProvider diskFileMapProvider,
+ ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[]
invListCmpFactories,
+ ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[]
tokenCmpFactories,
+ IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy
mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback
ioOpCallback, int[] invertedIndexFields,
+ int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[]
invertedIndexFieldsForNonBulkLoadOps,
+ boolean durable) throws IndexException {
super(virtualBufferCaches, diskInvIndexFactory.getBufferCache(),
fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker,
ioScheduler, ioOpCallback, filterFrameFactory,
filterManager, filterFields, durable);
@@ -137,13 +137,14 @@
for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
InMemoryInvertedIndex memInvIndex =
createInMemoryInvertedIndex(virtualBufferCache,
new
VirtualFreePageManager(virtualBufferCache.getNumPages()), i);
- BTree deleteKeysBTree = BTreeUtils.createBTree(virtualBufferCache,
new VirtualFreePageManager(
- virtualBufferCache.getNumPages()),
virtualBufferCache.getFileMapProvider(), invListTypeTraits,
- invListCmpFactories, BTreeLeafFrameType.REGULAR_NSM,
+ BTree deleteKeysBTree = BTreeUtils.createBTree(virtualBufferCache,
+ new
VirtualFreePageManager(virtualBufferCache.getNumPages()),
+ virtualBufferCache.getFileMapProvider(),
invListTypeTraits, invListCmpFactories,
+ BTreeLeafFrameType.REGULAR_NSM,
new FileReference(new File(fileManager.getBaseDir() +
"_virtual_del_" + i)));
LSMInvertedIndexMemoryComponent mutableComponent = new
LSMInvertedIndexMemoryComponent(memInvIndex,
- deleteKeysBTree, virtualBufferCache, i == 0 ? true :
false, filterFactory == null ? null
- : filterFactory.createLSMComponentFilter());
+ deleteKeysBTree, virtualBufferCache, i == 0 ? true : false,
+ filterFactory == null ? null :
filterFactory.createLSMComponentFilter());
memoryComponents.add(mutableComponent);
++i;
}
@@ -396,8 +397,8 @@
}
if (ctx.filterTuple != null) {
ctx.filterTuple.reset(tuple);
-
memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter()
- .update(ctx.filterTuple, ctx.filterCmp);
+
memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter().update(ctx.filterTuple,
+ ctx.filterCmp);
}
}
@@ -418,9 +419,8 @@
IIndexAccessor invIndexAccessor =
((LSMInvertedIndexMemoryComponent) component).getInvIndex()
.createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
indexAccessors.add(invIndexAccessor);
- IIndexAccessor deletedKeysAccessor =
((LSMInvertedIndexMemoryComponent) component)
-
.getDeletedKeysBTree().createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
+ IIndexAccessor deletedKeysAccessor =
((LSMInvertedIndexMemoryComponent) component).getDeletedKeysBTree()
+ .createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
deletedKeysBTreeAccessors.add(deletedKeysAccessor);
} else {
IIndexAccessor invIndexAccessor =
((LSMInvertedIndexDiskComponent) component).getInvIndex()
@@ -450,8 +450,8 @@
initState = new LSMInvertedIndexSearchCursorInitialState(keyCmp,
keysOnlyTuple, indexAccessors,
deletedKeysBTreeAccessors,
((LSMInvertedIndexMemoryComponent)
memoryComponents.get(currentMutableComponentId.get()))
- .getDeletedKeysBTree().getLeafFrameFactory(),
ictx, includeMutableComponent, lsmHarness,
- operationalComponents);
+ .getDeletedKeysBTree().getLeafFrameFactory(),
+ ictx, includeMutableComponent, lsmHarness,
operationalComponents);
} else {
LSMInvertedIndexMemoryComponent mutableComponent =
(LSMInvertedIndexMemoryComponent) memoryComponents
.get(currentMutableComponentId.get());
@@ -459,8 +459,9 @@
MultiComparator tokensAndKeysCmp =
MultiComparator.create(memInvIndex.getBTree().getComparatorFactories());
initState = new
LSMInvertedIndexRangeSearchCursorInitialState(tokensAndKeysCmp, keyCmp,
keysOnlyTuple,
((LSMInvertedIndexMemoryComponent)
memoryComponents.get(currentMutableComponentId.get()))
- .getDeletedKeysBTree().getLeafFrameFactory(),
includeMutableComponent, lsmHarness,
- indexAccessors, deletedKeysBTreeAccessors, pred,
operationalComponents);
+ .getDeletedKeysBTree().getLeafFrameFactory(),
+ includeMutableComponent, lsmHarness, indexAccessors,
deletedKeysBTreeAccessors, pred,
+ operationalComponents);
}
return initState;
}
@@ -488,8 +489,8 @@
opCtx.setOperation(IndexOperation.FLUSH);
opCtx.getComponentHolder().add(flushingComponent);
ioScheduler.scheduleOperation(new LSMInvertedIndexFlushOperation(
- new LSMInvertedIndexAccessor(lsmHarness, opCtx),
flushingComponent, componentFileRefs
- .getInsertIndexFileReference(),
componentFileRefs.getDeleteIndexFileReference(),
+ new LSMInvertedIndexAccessor(lsmHarness, opCtx),
flushingComponent,
+ componentFileRefs.getInsertIndexFileReference(),
componentFileRefs.getDeleteIndexFileReference(),
componentFileRefs.getBloomFilterFileReference(), callback,
fileManager.getBaseDir()));
}
@@ -500,7 +501,7 @@
// Create an inverted index instance to be bulk loaded.
LSMInvertedIndexDiskComponent component =
createDiskInvIndexComponent(componentFactory,
flushOp.getDictBTreeFlushTarget(),
flushOp.getDeletedKeysBTreeFlushTarget(),
- flushOp.getBloomFilterFlushTarget(), true);
+ flushOp.getBloomFilterTarget(), true);
IInvertedIndex diskInvertedIndex = component.getInvIndex();
// Create a scan cursor on the BTree underlying the in-memory inverted
index.
@@ -525,8 +526,8 @@
}
invIndexBulkLoader.end();
- IIndexAccessor deletedKeysBTreeAccessor =
flushingComponent.getDeletedKeysBTree().createAccessor(
- NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
+ IIndexAccessor deletedKeysBTreeAccessor =
flushingComponent.getDeletedKeysBTree()
+ .createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
IIndexCursor btreeCountingCursor = ((BTreeAccessor)
deletedKeysBTreeAccessor).createCountingSearchCursor();
deletedKeysBTreeAccessor.search(btreeCountingCursor, nullPred);
long numBTreeTuples = 0L;
@@ -585,10 +586,11 @@
@Override
public void scheduleMerge(ILSMIndexOperationContext ctx,
ILSMIOOperationCallback callback)
throws HyracksDataException, IndexException {
- LSMInvertedIndexOpContext ictx =
createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ LSMInvertedIndexOpContext ictx =
createOpContext(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
ictx.setOperation(IndexOperation.MERGE);
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
- IIndexCursor cursor = new LSMInvertedIndexRangeSearchCursor(ictx);
+ ITreeIndexCursor cursor = new LSMInvertedIndexRangeSearchCursor(ictx);
LSMInvertedIndexDiskComponent firstComponent =
(LSMInvertedIndexDiskComponent) mergingComponents.get(0);
OnDiskInvertedIndex firstInvIndex = (OnDiskInvertedIndex)
firstComponent.getInvIndex();
@@ -620,7 +622,7 @@
// Create an inverted index instance.
LSMInvertedIndexDiskComponent component =
createDiskInvIndexComponent(componentFactory,
mergeOp.getDictBTreeMergeTarget(),
mergeOp.getDeletedKeysBTreeMergeTarget(),
- mergeOp.getBloomFilterMergeTarget(), true);
+ mergeOp.getBloomFilterTarget(), true);
IInvertedIndex mergedDiskInvertedIndex = component.getInvIndex();
@@ -806,11 +808,10 @@
}
protected LSMInvertedIndexDiskComponent
createDiskInvIndexComponent(ILSMComponentFactory factory,
- FileReference dictBTreeFileRef, FileReference btreeFileRef,
FileReference bloomFilterFileRef, boolean create)
- throws HyracksDataException, IndexException {
- LSMInvertedIndexDiskComponent component =
(LSMInvertedIndexDiskComponent) factory
- .createLSMComponentInstance(new
LSMComponentFileReferences(dictBTreeFileRef, btreeFileRef,
- bloomFilterFileRef));
+ FileReference dictBTreeFileRef, FileReference btreeFileRef,
FileReference bloomFilterFileRef,
+ boolean create) throws HyracksDataException, IndexException {
+ LSMInvertedIndexDiskComponent component =
(LSMInvertedIndexDiskComponent) factory.createLSMComponentInstance(
+ new LSMComponentFileReferences(dictBTreeFileRef, btreeFileRef,
bloomFilterFileRef, null));
if (create) {
component.getInvIndex().create();
component.getDeletedKeysBTree().create();
diff --git
a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java
b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java
index 799a3da..0539c0a 100644
---
a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java
+++
b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java
@@ -30,14 +30,12 @@
private final IInvertedIndex invIndex;
private final BTree deletedKeysBTree;
- private final BloomFilter bloomFilter;
public LSMInvertedIndexDiskComponent(IInvertedIndex invIndex, BTree
deletedKeysBTree, BloomFilter bloomFilter,
ILSMComponentFilter filter) {
- super(filter);
+ super(bloomFilter, filter);
this.invIndex = invIndex;
this.deletedKeysBTree = deletedKeysBTree;
- this.bloomFilter = bloomFilter;
}
@Override
@@ -46,8 +44,7 @@
invIndex.destroy();
deletedKeysBTree.deactivate();
deletedKeysBTree.destroy();
- bloomFilter.deactivate();
- bloomFilter.destroy();
+ super.destroy();
}
public IInvertedIndex getInvIndex() {
@@ -58,16 +55,11 @@
return deletedKeysBTree;
}
- public BloomFilter getBloomFilter() {
- return bloomFilter;
- }
-
@Override
public long getComponentSize() {
- return ((OnDiskInvertedIndex)
invIndex).getInvListsFile().getFile().length()
+ return super.getComponentSize() + ((OnDiskInvertedIndex)
invIndex).getInvListsFile().getFile().length()
+ ((OnDiskInvertedIndex)
invIndex).getBTree().getFileReference().getFile().length()
- + deletedKeysBTree.getFileReference().getFile().length()
- + bloomFilter.getFileReference().getFile().length();
+ + deletedKeysBTree.getFileReference().getFile().length();
}
@Override
diff --git
a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
index 6c699da..f0c047f 100644
---
a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
+++
b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
@@ -66,7 +66,8 @@
}
};
- public LSMInvertedIndexFileManager(IFileMapProvider fileMapProvider,
FileReference file, BTreeFactory btreeFactory) {
+ public LSMInvertedIndexFileManager(IFileMapProvider fileMapProvider,
FileReference file,
+ BTreeFactory btreeFactory) {
super(fileMapProvider, file, null);
this.btreeFactory = btreeFactory;
}
@@ -77,8 +78,8 @@
String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a flush
return new LSMComponentFileReferences(createFlushFile(baseName +
SPLIT_STRING + DICT_BTREE_SUFFIX),
- createFlushFile(baseName + SPLIT_STRING +
DELETED_KEYS_BTREE_SUFFIX), createFlushFile(baseName
- + SPLIT_STRING + BLOOM_FILTER_STRING));
+ createFlushFile(baseName + SPLIT_STRING +
DELETED_KEYS_BTREE_SUFFIX),
+ createFlushFile(baseName + SPLIT_STRING +
BLOOM_FILTER_STRING), null);
}
@Override
@@ -90,8 +91,8 @@
String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING +
lastTimestampRange[1];
// Get the range of timestamps by taking the earliest and the latest
timestamps
return new LSMComponentFileReferences(createMergeFile(baseName +
SPLIT_STRING + DICT_BTREE_SUFFIX),
- createMergeFile(baseName + SPLIT_STRING +
DELETED_KEYS_BTREE_SUFFIX), createMergeFile(baseName
- + SPLIT_STRING + BLOOM_FILTER_STRING));
+ createMergeFile(baseName + SPLIT_STRING +
DELETED_KEYS_BTREE_SUFFIX),
+ createMergeFile(baseName + SPLIT_STRING +
BLOOM_FILTER_STRING), null);
}
@Override
@@ -131,8 +132,8 @@
if (allDictBTreeFiles.size() == 1 && allInvListsFiles.size() == 1 &&
allDeletedKeysBTreeFiles.size() == 1
&& allBloomFilterFiles.size() == 1) {
- validFiles.add(new
LSMComponentFileReferences(allDictBTreeFiles.get(0).fileRef,
allDeletedKeysBTreeFiles
- .get(0).fileRef, allBloomFilterFiles.get(0).fileRef));
+ validFiles.add(new
LSMComponentFileReferences(allDictBTreeFiles.get(0).fileRef,
+ allDeletedKeysBTreeFiles.get(0).fileRef,
allBloomFilterFiles.get(0).fileRef, null));
return validFiles;
}
@@ -182,7 +183,8 @@
invalidBloomFilterFile.delete();
} else {
// This scenario should not be possible.
- throw new HyracksDataException("Found LSM files with
overlapping but not contained timetamp intervals.");
+ throw new HyracksDataException(
+ "Found LSM files with overlapping but not contained
timetamp intervals.");
}
}
@@ -200,7 +202,7 @@
ComparableFileName cmpDeletedKeysBTreeFile =
deletedKeysBTreeIter.next();
ComparableFileName cmpBloomFilterFileName =
bloomFilterFileIter.next();
validFiles.add(new
LSMComponentFileReferences(cmpDictBTreeFile.fileRef,
cmpDeletedKeysBTreeFile.fileRef,
- cmpBloomFilterFileName.fileRef));
+ cmpBloomFilterFileName.fileRef, null));
}
return validFiles;
diff --git
a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
index b38d23a..5aff65d 100644
---
a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
+++
b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
@@ -19,64 +19,36 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.impls;
-import java.util.Collections;
-import java.util.HashSet;
import java.util.Set;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IODeviceHandle;
-import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMFlushOperation;
-public class LSMInvertedIndexFlushOperation implements ILSMIOOperation,
Comparable<LSMInvertedIndexFlushOperation> {
- private final ILSMIndexAccessorInternal accessor;
- private final ILSMComponent flushingComponent;
+public class LSMInvertedIndexFlushOperation extends AbstractLSMFlushOperation
+ implements Comparable<LSMInvertedIndexFlushOperation> {
+
private final FileReference dictBTreeFlushTarget;
private final FileReference deletedKeysBTreeFlushTarget;
- private final FileReference bloomFilterFlushTarget;
- private final ILSMIOOperationCallback callback;
- private final String indexIdentifier;
public LSMInvertedIndexFlushOperation(ILSMIndexAccessorInternal accessor,
ILSMComponent flushingComponent,
FileReference dictBTreeFlushTarget, FileReference
deletedKeysBTreeFlushTarget,
FileReference bloomFilterFlushTarget, ILSMIOOperationCallback
callback, String indexIdentifier) {
- this.accessor = accessor;
- this.flushingComponent = flushingComponent;
+ super(accessor, flushingComponent, bloomFilterFlushTarget, callback,
indexIdentifier);
this.dictBTreeFlushTarget = dictBTreeFlushTarget;
this.deletedKeysBTreeFlushTarget = deletedKeysBTreeFlushTarget;
- this.bloomFilterFlushTarget = bloomFilterFlushTarget;
- this.callback = callback;
- this.indexIdentifier = indexIdentifier;
- }
-
- @Override
- public Set<IODeviceHandle> getReadDevices() {
- return Collections.emptySet();
}
@Override
public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ Set<IODeviceHandle> devs = super.getWriteDevices();
devs.add(dictBTreeFlushTarget.getDeviceHandle());
devs.add(deletedKeysBTreeFlushTarget.getDeviceHandle());
- devs.add(bloomFilterFlushTarget.getDeviceHandle());
return devs;
- }
-
- @Override
- public Boolean call() throws HyracksDataException, IndexException {
- accessor.flush(this);
- return true;
- }
-
- @Override
- public ILSMIOOperationCallback getCallback() {
- return callback;
}
public FileReference getDictBTreeFlushTarget() {
@@ -85,24 +57,6 @@
public FileReference getDeletedKeysBTreeFlushTarget() {
return deletedKeysBTreeFlushTarget;
- }
-
- public FileReference getBloomFilterFlushTarget() {
- return bloomFilterFlushTarget;
- }
-
- public ILSMComponent getFlushingComponent() {
- return flushingComponent;
- }
-
- @Override
- public String getIndexUniqueIdentifier() {
- return indexIdentifier;
- }
-
- @Override
- public LSMIOOpertionType getIOOpertionType() {
- return LSMIOOpertionType.FLUSH;
}
@Override
diff --git
a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
index 0a368c6..2abd0a0 100644
---
a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
+++
b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
@@ -19,75 +19,48 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.impls;
-import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IODeviceHandle;
-import org.apache.hyracks.storage.am.common.api.IIndexCursor;
-import org.apache.hyracks.storage.am.common.api.IndexException;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMMergeOperation;
import
org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex;
-public class LSMInvertedIndexMergeOperation implements ILSMIOOperation {
- private final ILSMIndexAccessorInternal accessor;
- private final List<ILSMComponent> mergingComponents;
- private final IIndexCursor cursor;
+public class LSMInvertedIndexMergeOperation extends AbstractLSMMergeOperation {
private final FileReference dictBTreeMergeTarget;
private final FileReference deletedKeysBTreeMergeTarget;
- private final FileReference bloomFilterMergeTarget;
- private final ILSMIOOperationCallback callback;
- private final String indexIdentifier;
public LSMInvertedIndexMergeOperation(ILSMIndexAccessorInternal accessor,
List<ILSMComponent> mergingComponents,
- IIndexCursor cursor, FileReference dictBTreeMergeTarget,
FileReference deletedKeysBTreeMergeTarget,
+ ITreeIndexCursor cursor, FileReference dictBTreeMergeTarget,
FileReference deletedKeysBTreeMergeTarget,
FileReference bloomFilterMergeTarget, ILSMIOOperationCallback
callback, String indexIdentifier) {
- this.accessor = accessor;
- this.mergingComponents = mergingComponents;
- this.cursor = cursor;
+ super(accessor, mergingComponents, cursor, bloomFilterMergeTarget,
callback, indexIdentifier);
this.dictBTreeMergeTarget = dictBTreeMergeTarget;
this.deletedKeysBTreeMergeTarget = deletedKeysBTreeMergeTarget;
- this.bloomFilterMergeTarget = bloomFilterMergeTarget;
- this.callback = callback;
- this.indexIdentifier = indexIdentifier;
}
@Override
public Set<IODeviceHandle> getReadDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ Set<IODeviceHandle> devs = super.getReadDevices();
for (Object o : mergingComponents) {
LSMInvertedIndexDiskComponent component =
(LSMInvertedIndexDiskComponent) o;
OnDiskInvertedIndex invIndex = (OnDiskInvertedIndex)
component.getInvIndex();
devs.add(invIndex.getBTree().getFileReference().getDeviceHandle());
devs.add(component.getDeletedKeysBTree().getFileReference().getDeviceHandle());
-
devs.add(component.getBloomFilter().getFileReference().getDeviceHandle());
}
return devs;
}
@Override
public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>();
+ Set<IODeviceHandle> devs = super.getWriteDevices();
devs.add(dictBTreeMergeTarget.getDeviceHandle());
devs.add(deletedKeysBTreeMergeTarget.getDeviceHandle());
- devs.add(bloomFilterMergeTarget.getDeviceHandle());
return devs;
- }
-
- @Override
- public Boolean call() throws HyracksDataException, IndexException {
- accessor.merge(this);
- return true;
- }
-
- @Override
- public ILSMIOOperationCallback getCallback() {
- return callback;
}
public FileReference getDictBTreeMergeTarget() {
@@ -98,25 +71,4 @@
return deletedKeysBTreeMergeTarget;
}
- public FileReference getBloomFilterMergeTarget() {
- return bloomFilterMergeTarget;
- }
-
- public IIndexCursor getCursor() {
- return cursor;
- }
-
- public List<ILSMComponent> getMergingComponents() {
- return mergingComponents;
- }
-
- @Override
- public String getIndexUniqueIdentifier() {
- return indexIdentifier;
- }
-
- @Override
- public LSMIOOpertionType getIOOpertionType() {
- return LSMIOOpertionType.MERGE;
- }
}
\ No newline at end of file
diff --git
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index 533dfac..540ac0f 100644
---
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -101,15 +101,15 @@
for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
RTree memRTree = new RTree(virtualBufferCache,
virtualBufferCache.getFileMapProvider(),
new
VirtualFreePageManager(virtualBufferCache.getNumPages()),
rtreeInteriorFrameFactory,
- rtreeLeafFrameFactory, rtreeCmpFactories, fieldCount, new
FileReference(new File(
- fileManager.getBaseDir() + "_virtual_r_" + i)));
+ rtreeLeafFrameFactory, rtreeCmpFactories, fieldCount,
+ new FileReference(new File(fileManager.getBaseDir() +
"_virtual_r_" + i)));
BTree memBTree = new BTree(virtualBufferCache,
virtualBufferCache.getFileMapProvider(),
new
VirtualFreePageManager(virtualBufferCache.getNumPages()),
btreeInteriorFrameFactory,
- btreeLeafFrameFactory, btreeCmpFactories,
btreeCmpFactories.length, new FileReference(new File(
- fileManager.getBaseDir() + "_virtual_b_" + i)));
+ btreeLeafFrameFactory, btreeCmpFactories,
btreeCmpFactories.length,
+ new FileReference(new File(fileManager.getBaseDir() +
"_virtual_b_" + i)));
LSMRTreeMemoryComponent mutableComponent = new
LSMRTreeMemoryComponent(memRTree, memBTree,
- virtualBufferCache, i == 0 ? true : false, filterFactory
== null ? null
- : filterFactory.createLSMComponentFilter());
+ virtualBufferCache, i == 0 ? true : false,
+ filterFactory == null ? null :
filterFactory.createLSMComponentFilter());
memoryComponents.add(mutableComponent);
++i;
}
@@ -309,11 +309,10 @@
protected LSMRTreeDiskComponent createDiskComponent(ILSMComponentFactory
factory, FileReference insertFileRef,
FileReference deleteFileRef, FileReference bloomFilterFileRef,
boolean createComponent)
- throws HyracksDataException, IndexException {
+ throws HyracksDataException, IndexException {
// Create new tree instance.
- LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) factory
- .createLSMComponentInstance(new
LSMComponentFileReferences(insertFileRef, deleteFileRef,
- bloomFilterFileRef));
+ LSMRTreeDiskComponent component = (LSMRTreeDiskComponent)
factory.createLSMComponentInstance(
+ new LSMComponentFileReferences(insertFileRef, deleteFileRef,
bloomFilterFileRef, null));
if (createComponent) {
component.getRTree().create();
if (component.getBTree() != null) {
@@ -407,8 +406,8 @@
}
if (ctx.filterTuple != null) {
ctx.filterTuple.reset(tuple);
-
memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter()
- .update(ctx.filterTuple, ctx.filterCmp);
+
memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter().update(ctx.filterTuple,
+ ctx.filterCmp);
}
}
diff --git
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index f5be6c8..d963a54 100644
---
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -233,7 +233,7 @@
SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
LSMRTreeDiskComponent component =
createDiskComponent(componentFactory, flushOp.getRTreeFlushTarget(),
- flushOp.getBTreeFlushTarget(),
flushOp.getBloomFilterFlushTarget(), true);
+ flushOp.getBTreeFlushTarget(), flushOp.getBloomFilterTarget(),
true);
RTree diskRTree = component.getRTree();
IIndexBulkLoader rTreeBulkloader;
ITreeIndexCursor cursor;
diff --git
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java
index 4e6952b..b839b2c 100644
---
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java
+++
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java
@@ -28,13 +28,11 @@
public class LSMRTreeDiskComponent extends AbstractDiskLSMComponent {
private final RTree rtree;
private final BTree btree;
- private final BloomFilter bloomFilter;
public LSMRTreeDiskComponent(RTree rtree, BTree btree, BloomFilter
bloomFilter, ILSMComponentFilter filter) {
- super(filter);
+ super(bloomFilter, filter);
this.rtree = rtree;
this.btree = btree;
- this.bloomFilter = bloomFilter;
}
@Override
@@ -44,8 +42,7 @@
if (btree != null) {
btree.deactivate();
btree.destroy();
- bloomFilter.deactivate();
- bloomFilter.destroy();
+ super.destroy();
}
}
@@ -57,16 +54,12 @@
return btree;
}
- public BloomFilter getBloomFilter() {
- return bloomFilter;
- }
-
@Override
public long getComponentSize() {
long size = rtree.getFileReference().getFile().length();
if (btree != null) {
size += btree.getFileReference().getFile().length();
- size += bloomFilter.getFileReference().getFile().length();
+ size += super.getComponentSize();
}
return size;
}
diff --git
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
index 4f50980..5eda83f 100644
---
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
+++
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
@@ -73,8 +73,8 @@
String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a flush
return new LSMComponentFileReferences(createFlushFile(baseName +
SPLIT_STRING + RTREE_STRING),
- createFlushFile(baseName + SPLIT_STRING + BTREE_STRING),
createFlushFile(baseName + SPLIT_STRING
- + BLOOM_FILTER_STRING));
+ createFlushFile(baseName + SPLIT_STRING + BTREE_STRING),
+ createFlushFile(baseName + SPLIT_STRING +
BLOOM_FILTER_STRING), null);
}
@Override
@@ -87,8 +87,8 @@
// Get the range of timestamps by taking the earliest and the latest
// timestamps
return new LSMComponentFileReferences(createMergeFile(baseName +
SPLIT_STRING + RTREE_STRING),
- createMergeFile(baseName + SPLIT_STRING + BTREE_STRING),
createMergeFile(baseName + SPLIT_STRING
- + BLOOM_FILTER_STRING));
+ createMergeFile(baseName + SPLIT_STRING + BTREE_STRING),
+ createMergeFile(baseName + SPLIT_STRING +
BLOOM_FILTER_STRING), null);
}
@Override
@@ -109,7 +109,8 @@
btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
}
validateFiles(btreeFilesSet, allRTreeFiles,
getCompoundFilter(transactionFilter, rtreeFilter), rtreeFactory);
- validateFiles(btreeFilesSet, allBloomFilterFiles,
getCompoundFilter(transactionFilter, bloomFilterFilter), null);
+ validateFiles(btreeFilesSet, allBloomFilterFiles,
getCompoundFilter(transactionFilter, bloomFilterFilter),
+ null);
// Sanity check.
if (allRTreeFiles.size() != allBTreeFiles.size() ||
allBTreeFiles.size() != allBloomFilterFiles.size()) {
@@ -124,7 +125,7 @@
if (allRTreeFiles.size() == 1 && allBTreeFiles.size() == 1 &&
allBloomFilterFiles.size() == 1) {
validFiles.add(new
LSMComponentFileReferences(allRTreeFiles.get(0).fileRef,
allBTreeFiles.get(0).fileRef,
- allBloomFilterFiles.get(0).fileRef));
+ allBloomFilterFiles.get(0).fileRef, null));
return validFiles;
}
@@ -174,7 +175,8 @@
invalidBloomFilterFile.delete();
} else {
// This scenario should not be possible.
- throw new HyracksDataException("Found LSM files with
overlapping but not contained timetamp intervals.");
+ throw new HyracksDataException(
+ "Found LSM files with overlapping but not contained
timetamp intervals.");
}
}
@@ -192,7 +194,7 @@
ComparableFileName cmpBTreeFileName = btreeFileIter.next();
ComparableFileName cmpBloomFilterFileName =
bloomFilterFileIter.next();
validFiles.add(new
LSMComponentFileReferences(cmpRTreeFileName.fileRef, cmpBTreeFileName.fileRef,
- cmpBloomFilterFileName.fileRef));
+ cmpBloomFilterFileName.fileRef, null));
}
return validFiles;
}
@@ -205,8 +207,8 @@
String baseName = baseDir + ts + SPLIT_STRING + ts;
return new LSMComponentFileReferences(createFlushFile(baseName +
SPLIT_STRING + RTREE_STRING),
- createFlushFile(baseName + SPLIT_STRING + BTREE_STRING),
createFlushFile(baseName + SPLIT_STRING
- + BLOOM_FILTER_STRING));
+ createFlushFile(baseName + SPLIT_STRING + BTREE_STRING),
+ createFlushFile(baseName + SPLIT_STRING +
BLOOM_FILTER_STRING), null);
}
@Override
@@ -250,6 +252,6 @@
FileReference rTreeFileRef = new FileReference(rTreeFile);
FileReference bTreeFileRef = new FileReference(bTreeFile);
FileReference bloomFilterFileRef = new FileReference(bloomFilterFile);
- return new LSMComponentFileReferences(rTreeFileRef, bTreeFileRef,
bloomFilterFileRef);
+ return new LSMComponentFileReferences(rTreeFileRef, bTreeFileRef,
bloomFilterFileRef, null);
}
}
diff --git
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
index 31455ae..09abf66 100644
---
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
+++
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
@@ -18,44 +18,27 @@
*/
package org.apache.hyracks.storage.am.lsm.rtree.impls;
-import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IODeviceHandle;
-import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
+import
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMFlushOperation;
-public class LSMRTreeFlushOperation implements ILSMIOOperation,
Comparable<LSMRTreeFlushOperation> {
+public class LSMRTreeFlushOperation extends AbstractLSMFlushOperation
implements Comparable<LSMRTreeFlushOperation> {
- private final ILSMIndexAccessorInternal accessor;
- private final ILSMComponent flushingComponent;
private final FileReference rtreeFlushTarget;
private final FileReference btreeFlushTarget;
- private final FileReference bloomFilterFlushTarget;
- private final ILSMIOOperationCallback callback;
- private final String indexIdentifier;
public LSMRTreeFlushOperation(ILSMIndexAccessorInternal accessor,
ILSMComponent flushingComponent,
FileReference rtreeFlushTarget, FileReference btreeFlushTarget,
FileReference bloomFilterFlushTarget,
ILSMIOOperationCallback callback, String indexIdentifier) {
- this.accessor = accessor;
- this.flushingComponent = flushingComponent;
+ super(accessor, flushingComponent, bloomFilterFlushTarget, callback,
indexIdentifier);
this.rtreeFlushTarget = rtreeFlushTarget;
this.btreeFlushTarget = btreeFlushTarget;
- this.bloomFilterFlushTarget = bloomFilterFlushTarget;
- this.callback = callback;
- this.indexIdentifier = indexIdentifier;
- }
-
- @Override
- public Set<IODeviceHandle> getReadDevices() {
- return Collections.emptySet();
}
@Override
@@ -64,20 +47,9 @@
devs.add(rtreeFlushTarget.getDeviceHandle());
if (btreeFlushTarget != null) {
devs.add(btreeFlushTarget.getDeviceHandle());
- devs.add(bloomFilterFlushTarget.getDeviceHandle());
+ devs.add(bloomFilterTarget.getDeviceHandle());
}
return devs;
- }
-
- @Override
- public Boolean call() throws HyracksDataException, IndexException {
- accessor.flush(this);
- return true;
- }
-
- @Override
- public ILSMIOOperationCallback getCallback() {
- return callback;
}
public FileReference getRTreeFlushTarget() {
@@ -86,24 +58,6 @@
public FileReference getBTreeFlushTarget() {
return btreeFlushTarget;
- }
-
- public FileReference getBloomFilterFlushTarget() {
- return bloomFilterFlushTarget;
- }
-
- public ILSMComponent getFlushingComponent() {
- return flushingComponent;
- }
-
- @Override
- public String getIndexUniqueIdentifier() {
- return indexIdentifier;
- }
-
- @Override
- public LSMIOOpertionType getIOOpertionType() {
- return LSMIOOpertionType.FLUSH;
}
@Override
diff --git a/hyracks/pom.xml b/hyracks/pom.xml
index 6ed530a..f427b2c 100644
--- a/hyracks/pom.xml
+++ b/hyracks/pom.xml
@@ -40,6 +40,35 @@
<version>1.2</version>
</plugin>
</plugins>
+ <pluginManagement>
+ <plugins>
+ <!--This plugin's configuration is used to store Eclipse m2e
settings only. It has no influence on the Maven build itself.-->
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+
<artifactId>maven-plugin-plugin</artifactId>
+ <versionRange>[3.4,)</versionRange>
+ <goals>
+ <goal>descriptor</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore></ignore>
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
</build>
<repositories>
--
To view, visit https://asterix-gerrit.ics.uci.edu/453
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ifc8927a7fe90577d787c3d30c371ea95dd4aa24e
Gerrit-PatchSet: 1
Gerrit-Project: hyracks
Gerrit-Branch: master
Gerrit-Owner: Ildar Absalyamov <[email protected]>
