Repository: asterixdb Updated Branches: refs/heads/master 10e5ad1a7 -> f94fdcc20
[ASTERIXDB-2250] Clean up files after failed flush/merge - user model changes: no - storage format changes: no - interface changes: no Details: - Currently we didn't clean up component files if the flush/merge operation fails. As a result, when a failure happens, the subsequent retries must fail as well because the files already exist. - This patch cleans up component files when there is exception thrown during flush/merge operation - Added a test case on failed merge Change-Id: I94630613cfe68de9d5784e022ca3834de959aa02 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2300 Sonar-Qube: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Ian Maxon <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/f94fdcc2 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/f94fdcc2 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/f94fdcc2 Branch: refs/heads/master Commit: f94fdcc20489303db2bdb534d6352a7b1c71c87b Parents: 10e5ad1 Author: luochen01 <[email protected]> Authored: Wed Jan 31 08:59:20 2018 -0800 Committer: Luo Chen <[email protected]> Committed: Thu Feb 1 19:32:35 2018 -0800 ---------------------------------------------------------------------- .../lsm/btree/impls/LSMBTreeFlushOperation.java | 6 + .../lsm/btree/impls/LSMBTreeMergeOperation.java | 6 + .../impls/LSMBTreeWithBuddyMergeOperation.java | 5 + .../am/lsm/common/api/ILSMIOOperation.java | 6 + .../am/lsm/common/impls/AbstractLSMIndex.java | 70 ++++++++- .../am/lsm/common/impls/FlushOperation.java | 2 +- .../impls/LSMComponentFileReferences.java | 4 + .../am/lsm/common/impls/MergeOperation.java | 2 +- .../am/lsm/common/impls/TracedIOOperation.java | 6 + .../impls/LSMInvertedIndexFlushOperation.java | 6 + .../impls/LSMInvertedIndexMergeOperation.java | 6 + .../lsm/rtree/impls/LSMRTreeFlushOperation.java | 6 + .../lsm/rtree/impls/LSMRTreeMergeOperation.java | 6 + .../am/lsm/btree/LSMBTreeMergeFailTest.java | 157 +++++++++++++++++++ 14 files changed, 279 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f94fdcc2/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java index e3424e5..7acc59f 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFlushOperation.java @@ -22,6 +22,7 @@ import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; import org.apache.hyracks.storage.am.lsm.common.impls.FlushOperation; +import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; public class LSMBTreeFlushOperation extends FlushOperation { private final FileReference bloomFilterFlushTarget; @@ -35,4 +36,9 @@ public class LSMBTreeFlushOperation extends FlushOperation { public FileReference getBloomFilterTarget() { return bloomFilterFlushTarget; } + + @Override + public LSMComponentFileReferences getComponentFiles() { + return new LSMComponentFileReferences(target, null, bloomFilterFlushTarget); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f94fdcc2/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java index edfa7e1..3aef4c2 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMergeOperation.java @@ -22,6 +22,7 @@ package org.apache.hyracks.storage.am.lsm.btree.impls; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; +import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation; import org.apache.hyracks.storage.common.IIndexCursor; @@ -38,4 +39,9 @@ public class LSMBTreeMergeOperation extends MergeOperation { public FileReference getBloomFilterTarget() { return bloomFilterMergeTarget; } + + @Override + public LSMComponentFileReferences getComponentFiles() { + return new LSMComponentFileReferences(target, null, bloomFilterMergeTarget); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f94fdcc2/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java index 14cf778..dd4fcf5 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMergeOperation.java @@ -21,6 +21,7 @@ package org.apache.hyracks.storage.am.lsm.btree.impls; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; +import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation; import org.apache.hyracks.storage.common.IIndexCursor; @@ -51,4 +52,8 @@ public class LSMBTreeWithBuddyMergeOperation extends MergeOperation { return keepDeletedTuples; } + @Override + public LSMComponentFileReferences getComponentFiles() { + return new LSMComponentFileReferences(target, buddyBtreeMergeTarget, bloomFilterMergeTarget); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f94fdcc2/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java index ff32628..f5ee23b 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java @@ -23,6 +23,7 @@ import java.util.concurrent.Callable; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IODeviceHandle; +import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; public interface ILSMIOOperation extends Callable<Boolean> { @@ -67,4 +68,9 @@ public interface ILSMIOOperation extends Callable<Boolean> { * @return the accessor of the operation */ ILSMIndexAccessor getAccessor(); + + /** + * @return the component files produced by this operation + */ + LSMComponentFileReferences getComponentFiles(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f94fdcc2/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java index 749b3ba..6c1ef55 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java @@ -714,21 +714,77 @@ public abstract class AbstractLSMIndex implements ILSMIndex { ILSMIndexOperationContext opCtx = accessor.getOpContext(); if (opCtx.getOperation() == IndexOperation.DELETE_MEMORY_COMPONENT) { return EmptyComponent.INSTANCE; - } else { - if (LOGGER.isInfoEnabled()) { - FlushOperation flushOp = (FlushOperation) operation; - LOGGER.log(Level.INFO, "Flushing component with id: " + flushOp.getFlushingComponent().getId()); + } + if (LOGGER.isInfoEnabled()) { + FlushOperation flushOp = (FlushOperation) operation; + LOGGER.log(Level.INFO, "Flushing component with id: " + flushOp.getFlushingComponent().getId()); + } + ILSMDiskComponent component = null; + try { + component = doFlush(operation); + return component; + } catch (Exception e) { + LOGGER.error("Fail to execute flush " + this, e); + // clean up component + try { + cleanUpFiles(operation); + } catch (HyracksDataException e1) { + e.addSuppressed(e1); } - return doFlush(operation); + throw HyracksDataException.create(e); } + } @Override public final ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException { ILSMIndexAccessor accessor = operation.getAccessor(); ILSMIndexOperationContext opCtx = accessor.getOpContext(); - return opCtx.getOperation() == IndexOperation.DELETE_DISK_COMPONENTS ? EmptyComponent.INSTANCE - : doMerge(operation); + ILSMDiskComponent component = null; + try { + component = opCtx.getOperation() == IndexOperation.DELETE_DISK_COMPONENTS ? EmptyComponent.INSTANCE + : doMerge(operation); + return component; + } catch (Exception e) { + LOGGER.error("Fail to execute merge " + this, e); + // clean up component + try { + cleanUpFiles(operation); + } catch (HyracksDataException e1) { + e.addSuppressed(e1); + } + throw HyracksDataException.create(e); + } + + } + + protected void cleanUpFiles(ILSMIOOperation operation) throws HyracksDataException { + LSMComponentFileReferences componentFiles = operation.getComponentFiles(); + if (componentFiles == null) { + return; + } + FileReference[] files = componentFiles.getFileReferences(); + HyracksDataException exception = null; + for (FileReference file : files) { + try { + cleanUpFile(file); + } catch (HyracksDataException e) { + if (exception == null) { + exception = e; + } else { + exception.addSuppressed(e); + } + } + } + if (exception != null) { + throw exception; + } + } + + protected void cleanUpFile(FileReference file) throws HyracksDataException { + if (file != null) { + diskBufferCache.deleteFile(file); + } } protected abstract LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f94fdcc2/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java index e809925..d835021 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java @@ -27,7 +27,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; -public class FlushOperation extends AbstractIoOperation implements Comparable<ILSMIOOperation> { +public abstract class FlushOperation extends AbstractIoOperation implements Comparable<ILSMIOOperation> { public FlushOperation(ILSMIndexAccessor accessor, FileReference target, ILSMIOOperationCallback callback, String indexIdentifier) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f94fdcc2/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java index 8e98087..4dd57eb 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java @@ -49,4 +49,8 @@ public final class LSMComponentFileReferences { public FileReference getBloomFilterFileReference() { return bloomFilterFileReference; } + + public FileReference[] getFileReferences() { + return new FileReference[] { insertIndexFileReference, deleteIndexFileReference, bloomFilterFileReference }; + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f94fdcc2/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MergeOperation.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MergeOperation.java index e16da5b..ec2305d 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MergeOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MergeOperation.java @@ -27,7 +27,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; import org.apache.hyracks.storage.common.IIndexCursor; -public class MergeOperation extends AbstractIoOperation { +public abstract class MergeOperation extends AbstractIoOperation { protected final IIndexCursor cursor; public MergeOperation(ILSMIndexAccessor accessor, FileReference target, ILSMIOOperationCallback callback, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f94fdcc2/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java index 5fa6b4f..572e05c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java @@ -104,6 +104,11 @@ class TracedIOOperation implements ILSMIOOperation { public ILSMIndexAccessor getAccessor() { return ioOp.getAccessor(); } + + @Override + public LSMComponentFileReferences getComponentFiles() { + return ioOp.getComponentFiles(); + } } class ComparableTracedIOOperation extends TracedIOOperation implements Comparable<ILSMIOOperation> { @@ -132,4 +137,5 @@ class ComparableTracedIOOperation extends TracedIOOperation implements Comparabl + other.getClass().getSimpleName() + " in " + getClass().getSimpleName()); return Integer.signum(hashCode() - other.hashCode()); } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f94fdcc2/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java index 2106f6a..5b272ee 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java @@ -23,6 +23,7 @@ import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; import org.apache.hyracks.storage.am.lsm.common.impls.FlushOperation; +import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; public class LSMInvertedIndexFlushOperation extends FlushOperation { private final FileReference deletedKeysBTreeFlushTarget; @@ -43,4 +44,9 @@ public class LSMInvertedIndexFlushOperation extends FlushOperation { public FileReference getBloomFilterTarget() { return bloomFilterFlushTarget; } + + @Override + public LSMComponentFileReferences getComponentFiles() { + return new LSMComponentFileReferences(target, deletedKeysBTreeFlushTarget, bloomFilterFlushTarget); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f94fdcc2/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java index 2c1db0f..408d9bb 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java @@ -22,6 +22,7 @@ package org.apache.hyracks.storage.am.lsm.invertedindex.impls; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; +import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation; import org.apache.hyracks.storage.common.IIndexCursor; @@ -44,4 +45,9 @@ public class LSMInvertedIndexMergeOperation extends MergeOperation { public FileReference getBloomFilterTarget() { return bloomFilterMergeTarget; } + + @Override + public LSMComponentFileReferences getComponentFiles() { + return new LSMComponentFileReferences(target, deletedKeysBTreeMergeTarget, bloomFilterMergeTarget); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f94fdcc2/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java index 6991c56..2306ebd 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java @@ -22,6 +22,7 @@ import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; import org.apache.hyracks.storage.am.lsm.common.impls.FlushOperation; +import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; public class LSMRTreeFlushOperation extends FlushOperation { @@ -42,4 +43,9 @@ public class LSMRTreeFlushOperation extends FlushOperation { public FileReference getBloomFilterTarget() { return bloomFilterFlushTarget; } + + @Override + public LSMComponentFileReferences getComponentFiles() { + return new LSMComponentFileReferences(target, btreeFlushTarget, bloomFilterFlushTarget); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f94fdcc2/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java index 572ff01..aed5981 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java @@ -21,6 +21,7 @@ package org.apache.hyracks.storage.am.lsm.rtree.impls; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; +import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation; import org.apache.hyracks.storage.common.IIndexCursor; @@ -43,4 +44,9 @@ public class LSMRTreeMergeOperation extends MergeOperation { public FileReference getBloomFilterTarget() { return bloomFilterMergeTarget; } + + @Override + public LSMComponentFileReferences getComponentFiles() { + return new LSMComponentFileReferences(btreeMergeTarget, null, bloomFilterMergeTarget); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f94fdcc2/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeFailTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeFailTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeFailTest.java new file mode 100644 index 0000000..4c325c0 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeFailTest.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.storage.am.lsm.btree; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; + +import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; +import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer; +import org.apache.hyracks.dataflow.common.utils.TupleUtils; +import org.apache.hyracks.storage.am.btree.OrderedIndexTestUtils; +import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType; +import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters; +import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree; +import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeRangeSearchCursor; +import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext; +import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext; +import org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +@SuppressWarnings("rawtypes") +public class LSMBTreeMergeFailTest { + + private final OrderedIndexTestUtils orderedIndexTestUtils = new OrderedIndexTestUtils(); + + private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness(); + + private final TestIoScheduler scheduler = new TestIoScheduler(); + + @Before + public void setUp() throws HyracksDataException { + harness.setUp(); + } + + @After + public void tearDown() throws HyracksDataException { + harness.tearDown(); + } + + @Test + public void testMergeFail() throws Exception { + ISerializerDeserializer[] fieldSerdes = + { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE }; + LSMBTreeTestContext ctx = createTestContext(fieldSerdes, 1, BTreeLeafFrameType.REGULAR_NSM, true); + LSMBTree btree = (LSMBTree) ctx.getIndex(); + btree.create(); + btree.activate(); + ILSMIndexAccessor accessor = btree.createAccessor(NoOpIndexAccessParameters.INSTANCE); + + ITupleReference tuple1 = TupleUtils.createIntegerTuple(1, 1, 1); + accessor.insert(tuple1); + // flush component + accessor.scheduleFlush(btree.getIOOperationCallback()); + + ITupleReference tuple2 = TupleUtils.createIntegerTuple(2, 2, 2); + accessor.insert(tuple2); + // flush component + accessor.scheduleFlush(btree.getIOOperationCallback()); + + ITupleReference tuple3 = TupleUtils.createIntegerTuple(3, 3, 3); + accessor.insert(tuple3); + // flush component + accessor.scheduleFlush(btree.getIOOperationCallback()); + + scheduler.modify = true; + + boolean exceptionThrown = false; + try { + accessor.scheduleMerge(btree.getIOOperationCallback(), btree.getDiskComponents()); + } catch (HyracksDataException e) { + exceptionThrown = true; + } + Assert.assertTrue(exceptionThrown); + + scheduler.modify = false; + accessor.scheduleMerge(btree.getIOOperationCallback(), btree.getDiskComponents()); + Assert.assertEquals(1, btree.getDiskComponents().size()); + + btree.deactivate(); + btree.destroy(); + } + + protected LSMBTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys, + BTreeLeafFrameType leafType, boolean filtered) throws Exception { + return LSMBTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), + harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys, + harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(), + scheduler, harness.getIOOperationCallbackFactory(), harness.getMetadataPageManagerFactory(), filtered, + true, false); + } + + private class TestIoScheduler implements ILSMIOOperationScheduler { + boolean modify = false; + + @Override + public void scheduleOperation(ILSMIOOperation operation) throws HyracksDataException { + if (modify) { + try { + modifyOperation(operation); + } catch (Exception e) { + throw HyracksDataException.create(e); + } + } + operation.call(); + } + + private void modifyOperation(ILSMIOOperation operation) throws Exception { + if (!(operation instanceof MergeOperation)) { + return; + } + Field field = MergeOperation.class.getDeclaredField("cursor"); + field.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + LSMBTreeRangeSearchCursor originalCursor = (LSMBTreeRangeSearchCursor) field.get(operation); + field.set(operation, new TestCursor(originalCursor.getOpCtx())); + } + } + + private class TestCursor extends LSMBTreeRangeSearchCursor { + public TestCursor(ILSMIndexOperationContext opCtx) { + super(opCtx); + } + + @Override + public boolean hasNext() throws HyracksDataException { + throw new UnsupportedOperationException(); + } + } +}
