abdullah alamoudi has submitted this change and it was merged. Change subject: [ASTERIXDB-2414][STO] Fix name of merge files ......................................................................
[ASTERIXDB-2414][STO] Fix name of merge files - user model changes: no - storage format changes: no - interface changes: no Details: - A bug is found where merge file names are created incorrectly where start and end components are reversed. - The bug is fixed and an explicit check for the invariance was added. Change-Id: I861765bc0f293bdfdf0285f97884d536204fdb1e Reviewed-on: https://asterix-gerrit.ics.uci.edu/2820 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Contrib: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Luo Chen <[email protected]> Reviewed-by: Wail Alkowaileet <[email protected]> Reviewed-by: Ian Maxon <[email protected]> --- M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java 6 files changed, 27 insertions(+), 10 deletions(-) Approvals: Luo Chen: Looks good to me, but someone else must approve Anon. E. Moose #1000171: Jenkins: Verified; No violations found; ; Verified Wail Alkowaileet: Looks good to me, approved Ian Maxon: Looks good to me, approved diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java index 8be75be..2be5125 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java @@ -188,8 +188,8 @@ } } LSMBTreeRangeSearchCursor cursor = new LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples); - BTree firstBTree = ((LSMBTreeDiskComponent) mergingComponents.get(0)).getIndex(); - BTree lastBTree = ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1)).getIndex(); + BTree lastBTree = ((LSMBTreeDiskComponent) mergingComponents.get(0)).getIndex(); + BTree firstBTree = ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1)).getIndex(); FileReference firstFile = firstBTree.getFileReference(); FileReference lastFile = lastBTree.getFileReference(); LSMComponentFileReferences relMergeFileRefs = diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java index 5bcf30d..be91244 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java @@ -256,8 +256,8 @@ protected LSMComponentFileReferences getMergeTargetFileName(List<ILSMComponent> mergingDiskComponents) throws HyracksDataException { - BTree firstTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(0)).getIndex(); - BTree lastTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1)) + BTree lastTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(0)).getIndex(); + BTree firstTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1)) .getIndex(); FileReference firstFile = firstTree.getFileReference(); FileReference lastFile = lastTree.getFileReference(); diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java index 377a218..3d928a4 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java @@ -400,8 +400,8 @@ mergeCtx.getComponentHolder().addAll(mergingComponents); propagateMap(ctx, mergeCtx); mergingComponents.stream().forEach(mergeCtx.getComponentsToBeMerged()::add); - ILSMDiskComponent firstComponent = mergingComponents.get(0); - ILSMDiskComponent lastComponent = mergingComponents.get(mergingComponents.size() - 1); + ILSMDiskComponent lastComponent = mergingComponents.get(0); + ILSMDiskComponent firstComponent = mergingComponents.get(mergingComponents.size() - 1); LSMComponentFileReferences mergeFileRefs = getMergeFileReferences(firstComponent, lastComponent); ILSMIOOperation mergeOp = TracedIOOperation.wrap(createMergeOperation(mergeCtx, mergeFileRefs, ioOpCallback), tracer); diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java index 42d3f56..57fd01d 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java @@ -208,9 +208,14 @@ throws HyracksDataException { String[] firstTimestampRange = firstFileName.split(DELIMITER); String[] lastTimestampRange = lastFileName.split(DELIMITER); + String start = firstTimestampRange[0]; + String end = lastTimestampRange[1]; + if (end.compareTo(start) <= 0) { + throw new IllegalArgumentException( + "A Merge file must have end greater than start. Found end: " + end + " and start: " + start); + } // Get the range of timestamps by taking the earliest and the latest timestamps - return new LSMComponentFileReferences( - baseDir.getChild(firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]), null, null); + return new LSMComponentFileReferences(baseDir.getChild(start + DELIMITER + end), null, null); } @Override @@ -327,6 +332,12 @@ } return b.interval[1].compareTo(interval[1]); } + + @Override + public String toString() { + return "{\"type\" : \"" + (interval[0].equals(interval[1]) ? "flush" : "merge") + "\", \"start\" : \"" + + interval[0] + "\", \"end\" : \"" + interval[1] + "\"}"; + } } private class RecencyComparator implements Comparator<ComparableFileName> { diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java index 4dd57eb..9346b56 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java @@ -53,4 +53,10 @@ public FileReference[] getFileReferences() { return new FileReference[] { insertIndexFileReference, deleteIndexFileReference, bloomFilterFileReference }; } + + @Override + public String toString() { + return "{ \"insert\" : \"" + insertIndexFileReference + "\", \"delete\" : \"" + deleteIndexFileReference + + "\", \"bloom\" : \"" + bloomFilterFileReference + "\"}"; + } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java index 5e8bd0b..015f34c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java @@ -559,8 +559,8 @@ List<ILSMComponent> mergingComponents = ctx.getComponentHolder(); LSMRTreeSortedCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer, buddyBTreeFields); LSMComponentFileReferences relMergeFileRefs = - getMergeFileReferences((ILSMDiskComponent) mergingComponents.get(0), - (ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1)); + getMergeFileReferences((ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1), + (ILSMDiskComponent) mergingComponents.get(0)); ILSMIndexAccessor accessor = new LSMRTreeAccessor(getHarness(), rctx, buddyBTreeFields); // create the merge operation. LSMRTreeMergeOperation mergeOp = -- To view, visit https://asterix-gerrit.ics.uci.edu/2820 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I861765bc0f293bdfdf0285f97884d536204fdb1e Gerrit-PatchSet: 6 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: abdullah alamoudi <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Dmitry Lychagin <[email protected]> Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Luo Chen <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: Wail Alkowaileet <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
