abdullah alamoudi has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/2820
Change subject: [ASTERIXDB-2414][STO] Fix name of merge files
......................................................................
[ASTERIXDB-2414][STO] Fix name of merge files
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- A bug is found where merge file names are created incorrectly
where start and end components are reversed.
- The bug is fixed and an explicit check for the invariance was
added.
Change-Id: I861765bc0f293bdfdf0285f97884d536204fdb1e
---
M asterixdb/asterix-app/src/test/resources/cc.conf
M
hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
M
hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/FileMapManager.java
11 files changed, 53 insertions(+), 16 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/20/2820/1
diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf
b/asterixdb/asterix-app/src/test/resources/cc.conf
index fc95dd4..c8d8dee 100644
--- a/asterixdb/asterix-app/src/test/resources/cc.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc.conf
@@ -37,7 +37,7 @@
jvm.args=-Xmx4096m
-Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.pagesize=32KB
storage.buffercache.size=48MB
-storage.memorycomponent.globalbudget=512MB
+storage.memorycomponent.globalbudget=32MB
[cc]
address = 127.0.0.1
diff --git
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
index 4ded855..c5e1fdf 100644
---
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
+++
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
@@ -22,6 +22,8 @@
import java.io.Serializable;
import java.util.Date;
+import org.apache.hyracks.api.util.ExceptionUtils;
+
/**
* A device handle and a relative path.
* Used to identify a file in the local Node Controller.
@@ -33,6 +35,7 @@
private final IODeviceHandle dev;
private final String path;
private long registrationTime = 0L;
+ private Throwable registrationTrace;
public FileReference(IODeviceHandle dev, String path) {
file = new File(dev.getMount(), path);
@@ -99,6 +102,7 @@
"File " + toString() + " was already registered at " + new
Date(registrationTime));
}
registrationTime = System.currentTimeMillis();
+ registrationTrace =
ExceptionUtils.fromThreadStack(Thread.currentThread());
}
public long registrationTime() {
@@ -111,4 +115,8 @@
}
registrationTime = 0;
}
+
+ public Throwable registrationTrace() {
+ return registrationTrace;
+ }
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index 8be75be..2be5125 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -188,8 +188,8 @@
}
}
LSMBTreeRangeSearchCursor cursor = new
LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples);
- BTree firstBTree = ((LSMBTreeDiskComponent)
mergingComponents.get(0)).getIndex();
- BTree lastBTree = ((LSMBTreeDiskComponent)
mergingComponents.get(mergingComponents.size() - 1)).getIndex();
+ BTree lastBTree = ((LSMBTreeDiskComponent)
mergingComponents.get(0)).getIndex();
+ BTree firstBTree = ((LSMBTreeDiskComponent)
mergingComponents.get(mergingComponents.size() - 1)).getIndex();
FileReference firstFile = firstBTree.getFileReference();
FileReference lastFile = lastBTree.getFileReference();
LSMComponentFileReferences relMergeFileRefs =
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index 5bcf30d..be91244 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -256,8 +256,8 @@
protected LSMComponentFileReferences
getMergeTargetFileName(List<ILSMComponent> mergingDiskComponents)
throws HyracksDataException {
- BTree firstTree = ((LSMBTreeWithBuddyDiskComponent)
mergingDiskComponents.get(0)).getIndex();
- BTree lastTree = ((LSMBTreeWithBuddyDiskComponent)
mergingDiskComponents.get(mergingDiskComponents.size() - 1))
+ BTree lastTree = ((LSMBTreeWithBuddyDiskComponent)
mergingDiskComponents.get(0)).getIndex();
+ BTree firstTree = ((LSMBTreeWithBuddyDiskComponent)
mergingDiskComponents.get(mergingDiskComponents.size() - 1))
.getIndex();
FileReference firstFile = firstTree.getFileReference();
FileReference lastFile = lastTree.getFileReference();
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
index 54bc1fe..370c723 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
@@ -37,9 +37,12 @@
import
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
import
org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
+ private static final Logger LOGGER = LogManager.getLogger();
private static final FilenameFilter btreeFilter =
(dir, name) -> !name.startsWith(".") &&
name.endsWith(BTREE_SUFFIX);
private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
@@ -114,6 +117,7 @@
if (hasBloomFilter) {
Collections.sort(allBloomFilterFiles);
}
+ LOGGER.info("Sorted valid files: {}", allBloomFilterFiles);
List<ComparableFileName> validComparableBTreeFiles = new ArrayList<>();
ComparableFileName lastBTree = allBTreeFiles.get(0);
@@ -148,8 +152,10 @@
&& (!hasBloomFilter ||
(currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0
&&
currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0))) {
// Invalid files are completely contained in last interval.
+ LOGGER.info("Deleting {}", currentBTree.fullPath);
IoUtil.delete(new File(currentBTree.fullPath));
if (hasBloomFilter) {
+ LOGGER.info("Deleting {}", currentBloomFilter.fullPath);
IoUtil.delete(new File(currentBloomFilter.fullPath));
}
} else {
@@ -177,7 +183,7 @@
validFiles.add(new
LSMComponentFileReferences(cmpBTreeFileName.fileRef, null,
hasBloomFilter ? cmpBloomFilterFileName.fileRef : null));
}
-
+ LOGGER.info("Returning valid files {}", validFiles);
return validFiles;
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index 377a218..3d928a4 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -400,8 +400,8 @@
mergeCtx.getComponentHolder().addAll(mergingComponents);
propagateMap(ctx, mergeCtx);
mergingComponents.stream().forEach(mergeCtx.getComponentsToBeMerged()::add);
- ILSMDiskComponent firstComponent = mergingComponents.get(0);
- ILSMDiskComponent lastComponent =
mergingComponents.get(mergingComponents.size() - 1);
+ ILSMDiskComponent lastComponent = mergingComponents.get(0);
+ ILSMDiskComponent firstComponent =
mergingComponents.get(mergingComponents.size() - 1);
LSMComponentFileReferences mergeFileRefs =
getMergeFileReferences(firstComponent, lastComponent);
ILSMIOOperation mergeOp =
TracedIOOperation.wrap(createMergeOperation(mergeCtx,
mergeFileRefs, ioOpCallback), tracer);
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
index 59a919b..2e9550d 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
@@ -209,9 +209,14 @@
throws HyracksDataException {
String[] firstTimestampRange = firstFileName.split(DELIMITER);
String[] lastTimestampRange = lastFileName.split(DELIMITER);
+ String start = firstTimestampRange[0];
+ String end = lastTimestampRange[1];
+ if (end.compareTo(start) <= 0) {
+ throw new IllegalArgumentException(
+ "A Merge file must have end greater than start. Found end:
" + end + " and start: " + start);
+ }
// Get the range of timestamps by taking the earliest and the latest
timestamps
- return new LSMComponentFileReferences(
- baseDir.getChild(firstTimestampRange[0] + DELIMITER +
lastTimestampRange[1]), null, null);
+ return new LSMComponentFileReferences(baseDir.getChild(start +
DELIMITER + end), null, null);
}
@Override
@@ -328,6 +333,12 @@
}
return b.interval[1].compareTo(interval[1]);
}
+
+ @Override
+ public String toString() {
+ return "{\"type\" : \"" + (interval[0].equals(interval[1]) ?
"flush" : "merge") + "\", \"start\" : \""
+ + interval[0] + "\", \"end\" : \"" + interval[1] + "\"}";
+ }
}
private class RecencyComparator implements Comparator<ComparableFileName> {
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java
index 4dd57eb..9346b56 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFileReferences.java
@@ -53,4 +53,10 @@
public FileReference[] getFileReferences() {
return new FileReference[] { insertIndexFileReference,
deleteIndexFileReference, bloomFilterFileReference };
}
+
+ @Override
+ public String toString() {
+ return "{ \"insert\" : \"" + insertIndexFileReference + "\",
\"delete\" : \"" + deleteIndexFileReference
+ + "\", \"bloom\" : \"" + bloomFilterFileReference + "\"}";
+ }
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 3eea0a9..1cd5a93 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -228,8 +228,9 @@
if (!inactiveDiskComponents.isEmpty()) {
for (ILSMDiskComponent inactiveComp :
inactiveDiskComponents) {
if (inactiveComp.getFileReferenceCount() == 1) {
- inactiveDiskComponentsToBeDeleted =
inactiveDiskComponentsToBeDeleted == null
- ? new LinkedList<>() :
inactiveDiskComponentsToBeDeleted;
+ inactiveDiskComponentsToBeDeleted =
+ inactiveDiskComponentsToBeDeleted ==
null ? new LinkedList<>()
+ :
inactiveDiskComponentsToBeDeleted;
inactiveDiskComponentsToBeDeleted.add(inactiveComp);
}
}
@@ -532,8 +533,9 @@
public void doIo(ILSMIOOperation operation) {
try {
operation.getCallback().beforeOperation(operation);
- ILSMDiskComponent newComponent = operation.getIOOpertionType() ==
LSMIOOperationType.FLUSH
- ? lsmIndex.flush(operation) : lsmIndex.merge(operation);
+ ILSMDiskComponent newComponent =
+ operation.getIOOpertionType() == LSMIOOperationType.FLUSH
? lsmIndex.flush(operation)
+ : lsmIndex.merge(operation);
operation.setNewComponent(newComponent);
operation.getCallback().afterOperation(operation);
if (newComponent != null) {
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
index 5e8bd0b..015f34c 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
@@ -559,8 +559,8 @@
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
LSMRTreeSortedCursor cursor = new LSMRTreeSortedCursor(rctx,
linearizer, buddyBTreeFields);
LSMComponentFileReferences relMergeFileRefs =
- getMergeFileReferences((ILSMDiskComponent)
mergingComponents.get(0),
- (ILSMDiskComponent)
mergingComponents.get(mergingComponents.size() - 1));
+ getMergeFileReferences((ILSMDiskComponent)
mergingComponents.get(mergingComponents.size() - 1),
+ (ILSMDiskComponent) mergingComponents.get(0));
ILSMIndexAccessor accessor = new LSMRTreeAccessor(getHarness(), rctx,
buddyBTreeFields);
// create the merge operation.
LSMRTreeMergeOperation mergeOp =
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/FileMapManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/FileMapManager.java
index 32922d2..7b72df9 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/FileMapManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/FileMapManager.java
@@ -25,9 +25,12 @@
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class FileMapManager implements IFileMapManager {
private static final long serialVersionUID = 1L;
+ private static final Logger LOGGER = LogManager.getLogger();
private Map<Integer, FileReference> id2nameMap = new HashMap<>();
private Map<FileReference, Integer> name2IdMap = new HashMap<>();
@@ -77,6 +80,7 @@
Integer existingKey = name2IdMap.get(fileRef);
if (existingKey != null) {
FileReference prevFile = id2nameMap.get(existingKey);
+ LOGGER.warn("File {} Registered on", prevFile,
prevFile.registrationTrace());
throw HyracksDataException.create(ErrorCode.FILE_ALREADY_MAPPED,
fileRef, prevFile,
new Date(prevFile.registrationTime()).toString());
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/2820
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I861765bc0f293bdfdf0285f97884d536204fdb1e
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <[email protected]>
