Repository: asterixdb Updated Branches: refs/heads/master ad4761481 -> a731afb17
[ASTERIXDB-2414][STO] Fix name of merge files - user model changes: no - storage format changes: no - interface changes: no Details: - A bug is found where merge file names are created incorrectly where start and end components are reversed. - The bug is fixed and an explicit check for the invariance was added. Change-Id: I861765bc0f293bdfdf0285f97884d536204fdb1e Reviewed-on: https://asterix-gerrit.ics.uci.edu/2820 Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Luo Chen <cl...@uci.edu> Reviewed-by: Wail Alkowaileet <wael....@gmail.com> Reviewed-by: Ian Maxon <ima...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/a731afb1 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/a731afb1 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/a731afb1 Branch: refs/heads/master Commit: a731afb1724644b3f553662ac4cd97004adbd5eb Parents: ad47614 Author: Abdullah Alamoudi <bamou...@gmail.com> Authored: Thu Jul 26 22:14:08 2018 -0700 Committer: abdullah alamoudi <bamou...@gmail.com> Committed: Mon Jul 30 14:36:51 2018 -0700 ---------------------------------------------------------------------- .../storage/am/lsm/btree/impls/ExternalBTree.java | 4 ++-- .../am/lsm/btree/impls/ExternalBTreeWithBuddy.java | 4 ++-- .../am/lsm/common/impls/AbstractLSMIndex.java | 4 ++-- .../common/impls/AbstractLSMIndexFileManager.java | 15 +++++++++++++-- .../lsm/common/impls/LSMComponentFileReferences.java | 6 ++++++ .../storage/am/lsm/rtree/impls/ExternalRTree.java | 4 ++-- 6 files changed, 27 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a731afb1/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java index 8be75be..2be5125 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java @@ -188,8 +188,8 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { } } LSMBTreeRangeSearchCursor cursor = new LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples); - BTree firstBTree = ((LSMBTreeDiskComponent) mergingComponents.get(0)).getIndex(); - BTree lastBTree = ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1)).getIndex(); + BTree lastBTree = ((LSMBTreeDiskComponent) mergingComponents.get(0)).getIndex(); + BTree firstBTree = ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1)).getIndex(); FileReference firstFile = firstBTree.getFileReference(); FileReference lastFile = lastBTree.getFileReference(); LSMComponentFileReferences relMergeFileRefs = http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a731afb1/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java index 5bcf30d..be91244 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java @@ -256,8 +256,8 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd protected LSMComponentFileReferences getMergeTargetFileName(List<ILSMComponent> mergingDiskComponents) throws HyracksDataException { - BTree firstTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(0)).getIndex(); - BTree lastTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1)) + BTree lastTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(0)).getIndex(); + BTree firstTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1)) .getIndex(); FileReference firstFile = firstTree.getFileReference(); FileReference lastFile = lastTree.getFileReference(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a731afb1/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java index 377a218..3d928a4 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java @@ -400,8 +400,8 @@ public abstract class AbstractLSMIndex implements ILSMIndex { mergeCtx.getComponentHolder().addAll(mergingComponents); propagateMap(ctx, mergeCtx); mergingComponents.stream().forEach(mergeCtx.getComponentsToBeMerged()::add); - ILSMDiskComponent firstComponent = mergingComponents.get(0); - ILSMDiskComponent lastComponent = mergingComponents.get(mergingComponents.size() - 1); + ILSMDiskComponent lastComponent = mergingComponents.get(0); + ILSMDiskComponent firstComponent = mergingComponents.get(mergingComponents.size() - 1); LSMComponentFileReferences mergeFileRefs = getMergeFileReferences(firstComponent, lastComponent); ILSMIOOperation mergeOp = TracedIOOperation.wrap(createMergeOperation(mergeCtx, mergeFileRefs, ioOpCallback), tracer); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a731afb1/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java index 42d3f56..57fd01d 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java @@ -208,9 +208,14 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage throws HyracksDataException { String[] firstTimestampRange = firstFileName.split(DELIMITER); String[] lastTimestampRange = lastFileName.split(DELIMITER); + String start = firstTimestampRange[0]; + String end = lastTimestampRange[1]; + if (end.compareTo(start) <= 0) { + throw new IllegalArgumentException( + "A Merge file must have end greater than start. Found end: " + end + " and start: " + start); + } // Get the range of timestamps by taking the earliest and the latest timestamps - return new LSMComponentFileReferences( - baseDir.getChild(firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]), null, null); + return new LSMComponentFileReferences(baseDir.getChild(start + DELIMITER + end), null, null); } @Override @@ -327,6 +332,12 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage } return b.interval[1].compareTo(interval[1]); } + + @Override + public String toString() { + return "{\"type\" : \"" + (interval[0].equals(interval[1]) ? "flush" : "merge") + "\", \"start\" : \"" + + interval[0] + "\", \"end\" : \"" + interval[1] + "\"}"; + } } private class RecencyComparator implements Comparator<ComparableFileName> { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a731afb1/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java index 4dd57eb..9346b56 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java @@ -53,4 +53,10 @@ public final class LSMComponentFileReferences { public FileReference[] getFileReferences() { return new FileReference[] { insertIndexFileReference, deleteIndexFileReference, bloomFilterFileReference }; } + + @Override + public String toString() { + return "{ \"insert\" : \"" + insertIndexFileReference + "\", \"delete\" : \"" + deleteIndexFileReference + + "\", \"bloom\" : \"" + bloomFilterFileReference + "\"}"; + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a731afb1/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java index 5e8bd0b..015f34c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java @@ -559,8 +559,8 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { List<ILSMComponent> mergingComponents = ctx.getComponentHolder(); LSMRTreeSortedCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer, buddyBTreeFields); LSMComponentFileReferences relMergeFileRefs = - getMergeFileReferences((ILSMDiskComponent) mergingComponents.get(0), - (ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1)); + getMergeFileReferences((ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1), + (ILSMDiskComponent) mergingComponents.get(0)); ILSMIndexAccessor accessor = new LSMRTreeAccessor(getHarness(), rctx, buddyBTreeFields); // create the merge operation. LSMRTreeMergeOperation mergeOp =
