abdullah alamoudi has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/736
Change subject: Fixed Bugs Caused by Disabling Installer Tests
......................................................................
Fixed Bugs Caused by Disabling Installer Tests
Change-Id: Ic77f4ee76a77f4b49cac6835bbb976eb9dc92d3a
---
M
hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
M
hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
M
hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
M
hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
M
hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
5 files changed, 98 insertions(+), 81 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/hyracks refs/changes/36/736/1
diff --git
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
index 6e0beb3..4756825 100644
---
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
+++
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
@@ -35,18 +35,18 @@
private final int[] buddyBtreeFields;
private final int version;
- public ExternalBTreeWithBuddyDataflowHelper(IIndexOperatorDescriptor
opDesc, IHyracksTaskContext ctx,
- int partition, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory,
+ public ExternalBTreeWithBuddyDataflowHelper(IIndexOperatorDescriptor
opDesc, IHyracksTaskContext ctx, int partition,
+ ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider
opTrackerFactory,
ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory,
int[] buddyBtreeFields, int version, boolean durable) {
- super(opDesc, ctx, partition, null, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackFactory, null,
- null, null, durable);
+ super(opDesc, ctx, partition, null, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackFactory, null, null,
+ null, durable);
this.buddyBtreeFields = buddyBtreeFields;
this.version = version;
}
- public ExternalBTreeWithBuddyDataflowHelper(IIndexOperatorDescriptor
opDesc, IHyracksTaskContext ctx,
- int partition, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy,
+ public ExternalBTreeWithBuddyDataflowHelper(IIndexOperatorDescriptor
opDesc, IHyracksTaskContext ctx, int partition,
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[]
buddyBtreeFields, int version, boolean durable) {
super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate,
mergePolicy, opTrackerFactory, ioScheduler,
@@ -72,10 +72,10 @@
@Override
protected IIndex createIndexInstance() throws HyracksDataException {
AbstractTreeIndexOperatorDescriptor treeOpDesc =
(AbstractTreeIndexOperatorDescriptor) opDesc;
- return LSMBTreeUtils.createExternalBTreeWithBuddy(file,
opDesc.getStorageManager().getBufferCache(ctx), opDesc
- .getStorageManager().getFileMapProvider(ctx),
treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc
- .getTreeIndexComparatorFactories(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory
- .getOperationTracker(ctx), ioScheduler,
ioOpCallbackFactory.createIOOperationCallback(),
+ return LSMBTreeUtils.createExternalBTreeWithBuddy(file,
opDesc.getStorageManager().getBufferCache(ctx),
+ opDesc.getStorageManager().getFileMapProvider(ctx),
treeOpDesc.getTreeIndexTypeTraits(),
+ treeOpDesc.getTreeIndexComparatorFactories(),
bloomFilterFalsePositiveRate, mergePolicy,
+ opTrackerFactory.getOperationTracker(ctx), ioScheduler,
ioOpCallbackFactory.createIOOperationCallback(),
buddyBtreeFields, version, durable);
}
diff --git
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index 959e356..c9b8896 100644
---
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -319,7 +319,7 @@
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
BTree btree = component.getBTree();
BloomFilter bloomFilter = component.getBloomFilter();
- btree.deactivate();
+ btree.deactivateCloseHandle();
bloomFilter.deactivate();
}
for (ILSMComponent c : secondDiskComponents) {
@@ -328,7 +328,7 @@
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
BTree btree = component.getBTree();
BloomFilter bloomFilter = component.getBloomFilter();
- btree.deactivate();
+ btree.deactivateCloseHandle();
bloomFilter.deactivate();
}
}
diff --git
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index a4f355f..a40235a 100644
---
a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -35,7 +35,19 @@
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeBulkLoader;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
-import org.apache.hyracks.storage.am.common.api.*;
+import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
+import org.apache.hyracks.storage.am.common.api.IIndexCursor;
+import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
+import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
+import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
+import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
+import org.apache.hyracks.storage.am.common.api.ITreeIndex;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.common.api.ITwoPCIndexBulkLoader;
+import org.apache.hyracks.storage.am.common.api.IndexException;
+import org.apache.hyracks.storage.am.common.api.TreeIndexException;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
@@ -538,8 +550,8 @@
BTree btree = component.getBTree();
BTree buddyBtree = component.getBuddyBTree();
BloomFilter bloomFilter = component.getBloomFilter();
- btree.deactivate();
- buddyBtree.deactivate();
+ btree.deactivateCloseHandle();
+ buddyBtree.deactivateCloseHandle();
bloomFilter.deactivate();
}
for (ILSMComponent c : secondDiskComponents) {
@@ -549,8 +561,8 @@
BTree btree = component.getBTree();
BTree buddyBtree = component.getBuddyBTree();
BloomFilter bloomFilter = component.getBloomFilter();
- btree.deactivate();
- buddyBtree.deactivate();
+ btree.deactivateCloseHandle();
+ buddyBtree.deactivateCloseHandle();
bloomFilter.deactivate();
}
}
@@ -602,9 +614,13 @@
LSMBTreeWithBuddyDiskComponent component =
(LSMBTreeWithBuddyDiskComponent) factory.createLSMComponentInstance(
new LSMComponentFileReferences(insertFileRef, deleteFileRef,
bloomFilterFileRef));
if (createComponent) {
+ component.getBTree().create();
+ component.getBuddyBTree().create();
component.getBloomFilter().create();
+ } else {
+ component.getBTree().activate();
+ component.getBuddyBTree().activate();
}
-
component.getBloomFilter().activate();
return component;
}
diff --git
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
index 492af2d..51e0fbd 100644
---
a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
+++
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
@@ -357,8 +357,8 @@
RTree rtree = component.getRTree();
BTree btree = component.getBTree();
BloomFilter bloomFilter = component.getBloomFilter();
- rtree.deactivate();
- btree.deactivate();
+ rtree.deactivateCloseHandle();
+ btree.deactivateCloseHandle();
bloomFilter.deactivate();
}
for (ILSMComponent c : secondDiskComponents) {
@@ -368,8 +368,8 @@
RTree rtree = component.getRTree();
BTree btree = component.getBTree();
BloomFilter bloomFilter = component.getBloomFilter();
- rtree.deactivate();
- btree.deactivate();
+ rtree.deactivateCloseHandle();
+ btree.deactivateCloseHandle();
bloomFilter.deactivate();
}
}
diff --git
a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
index 27d0423..66dc866 100644
---
a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
+++
b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
@@ -20,7 +20,13 @@
import java.io.IOException;
import java.io.OutputStream;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
@@ -92,7 +98,7 @@
closed = false;
fifoWriter = new AsyncFIFOPageQueueManager(this);
- if( DEBUG ) {
+ if (DEBUG) {
confiscatedPages = new ArrayList<CachedPage>();
confiscatedPagesOwner = new HashMap<CachedPage,
StackTraceElement[]>();
confiscateLock = new ReentrantLock();
@@ -178,12 +184,12 @@
try {
for (CachedPage c : confiscatedPages) {
if (c.dpid == dpid && c.confiscated.get()) {
- while(confiscatedPages.contains(c)){
+ while (confiscatedPages.contains(c)) {
throw new IllegalStateException();
}
}
}
- }finally{
+ } finally {
confiscateLock.unlock();
}
}
@@ -199,13 +205,11 @@
cPage.valid = true;
}
pageReplacementStrategy.notifyCachePageAccess(cPage);
- if(DEBUG){
- pinnedPageOwner.put((CachedPage) cPage,
Thread.currentThread().getStackTrace());
+ if (DEBUG) {
+ pinnedPageOwner.put(cPage, Thread.currentThread().getStackTrace());
}
return cPage;
}
-
-
private boolean isVirtual(long vpid) throws HyracksDataException {
CachedPage virtPage = findPage(vpid, true);
@@ -227,11 +231,11 @@
try {
cPage = bucket.cachedPage;
while (cPage != null) {
- if(DEBUG) {
+ if (DEBUG) {
assert bucket.cachedPage != bucket.cachedPage.next;
}
if (cPage.dpid == dpid) {
- if(DEBUG) {
+ if (DEBUG) {
assert !cPage.confiscated.get();
}
cPage.pinCount.incrementAndGet();
@@ -285,11 +289,11 @@
try {
if (DEBUG) {
confiscateLock.lock();
- try{
+ try {
if (confiscatedPages.contains(victim)) {
throw new IllegalStateException();
}
- } finally{
+ } finally {
confiscateLock.unlock();
}
}
@@ -298,7 +302,7 @@
if (cPage.dpid == dpid) {
cPage.pinCount.incrementAndGet();
victim.pinCount.decrementAndGet();
- if(DEBUG) {
+ if (DEBUG) {
assert !cPage.confiscated.get();
}
return cPage;
@@ -312,7 +316,7 @@
bucket.bucketLock.unlock();
}
- if(DEBUG) {
+ if (DEBUG) {
assert !victim.confiscated.get();
}
return victim;
@@ -330,11 +334,11 @@
}
if (DEBUG) {
confiscateLock.lock();
- try{
+ try {
if (confiscatedPages.contains(victim)) {
throw new IllegalStateException();
}
- }finally{
+ } finally {
confiscateLock.unlock();
}
}
@@ -343,7 +347,7 @@
if (cPage.dpid == dpid) {
cPage.pinCount.incrementAndGet();
victim.pinCount.decrementAndGet();
- if(DEBUG) {
+ if (DEBUG) {
assert !victim.confiscated.get();
}
return cPage;
@@ -354,7 +358,7 @@
} finally {
bucket.bucketLock.unlock();
}
- if(DEBUG) {
+ if (DEBUG) {
assert !victim.confiscated.get();
}
return victim;
@@ -385,7 +389,7 @@
if (cPage.dpid == dpid) {
cPage.pinCount.incrementAndGet();
victim.pinCount.decrementAndGet();
- if(DEBUG) {
+ if (DEBUG) {
assert !cPage.confiscated.get();
}
return cPage;
@@ -399,7 +403,7 @@
while (victimPrev != null && victimPrev.next !=
victim) {
victimPrev = victimPrev.next;
}
- if(DEBUG) {
+ if (DEBUG) {
assert victimPrev != null;
}
victimPrev.next = victim.next;
@@ -411,7 +415,7 @@
victimBucket.bucketLock.unlock();
bucket.bucketLock.unlock();
}
- if(DEBUG) {
+ if (DEBUG) {
assert !victim.confiscated.get();
}
return victim;
@@ -470,12 +474,11 @@
}
}
buffer.append("Number of cached pages:
").append(nCachedPages).append('\n');
- if(DEBUG){
+ if (DEBUG) {
confiscateLock.lock();
- try{
+ try {
buffer.append("Number of confiscated pages:
").append(confiscatedPages.size()).append('\n');
- }
- finally{
+ } finally {
confiscateLock.unlock();
}
}
@@ -483,29 +486,28 @@
}
@Override
- public boolean isClean(){
+ public boolean isClean() {
List<Long> reachableDpids = new LinkedList<>();
synchronized (cachedPages) {
for (ICachedPageInternal internalPage : cachedPages) {
- CachedPage c = (CachedPage) internalPage;
- if (c.confiscated() ||
- c.latch.getReadLockCount() != 0 ||
c.latch.getWriteHoldCount() != 0) {
+ CachedPage c = (CachedPage) internalPage;
+ if (c.confiscated() || c.latch.getReadLockCount() != 0 ||
c.latch.getWriteHoldCount() != 0) {
return false;
}
- if(c.valid){
+ if (c.valid) {
reachableDpids.add(c.dpid);
}
}
}
- for(Long l: reachableDpids){
- if(!canFindValidCachedPage(l)){
+ for (Long l : reachableDpids) {
+ if (!canFindValidCachedPage(l)) {
return false;
}
}
return true;
}
- private boolean canFindValidCachedPage(long dpid){
+ private boolean canFindValidCachedPage(long dpid) {
int hash = hash(dpid);
CachedPage cPage = null;
CacheBucket bucket = pageMap[hash];
@@ -564,7 +566,7 @@
if (closed) {
throw new HyracksDataException("unpin called on a closed cache");
}
- if(DEBUG){
+ if (DEBUG) {
pinnedPageOwner.remove(page);
}
((CachedPage) page).pinCount.decrementAndGet();
@@ -967,7 +969,7 @@
if (fInfo == null) {
throw new HyracksDataException("No such file mapped for
fileId:" + fileId);
}
- if(DEBUG) {
+ if (DEBUG) {
assert ioManager.getSize(fInfo.getFileHandle()) %
getPageSize() == 0;
}
return (int) (ioManager.getSize(fInfo.getFileHandle()) /
getPageSize());
@@ -986,7 +988,7 @@
ICachedPage returnPage = null;
CachedPage victim = (CachedPage)
pageReplacementStrategy.findVictim();
if (victim != null) {
- if(DEBUG) {
+ if (DEBUG) {
assert !victim.confiscated.get();
}
// find a page that would possibly be evicted anyway
@@ -1012,14 +1014,14 @@
//traverse the bucket's linked list to find the victim.
while (curr != null) {
if (curr == victim) { // we found where the victim
- // resides in the hash table
+ // resides in the hash
table
if (victim.pinCount.get() != 1) {
victim.pinCount.decrementAndGet();
break;
}
// if this is the first page in the bucket
if (prev == null) {
- if(DEBUG) {
+ if (DEBUG) {
assert curr != curr.next;
}
bucket.cachedPage = bucket.cachedPage.next;
@@ -1028,12 +1030,12 @@
// if it isn't we need to make the previous
// node point to where it should
} else {
- if(DEBUG) {
+ if (DEBUG) {
assert curr.next != curr;
}
prev.next = curr.next;
curr.next = null;
- if(DEBUG) {
+ if (DEBUG) {
assert prev.next != prev;
}
found = true;
@@ -1058,11 +1060,10 @@
((CachedPage) returnPage).confiscated.set(true);
if (DEBUG) {
confiscateLock.lock();
- try{
+ try {
confiscatedPages.add((CachedPage) returnPage);
confiscatedPagesOwner.put((CachedPage) returnPage,
Thread.currentThread().getStackTrace());
- }
- finally{
+ } finally {
confiscateLock.unlock();
}
}
@@ -1098,14 +1099,14 @@
public void returnPage(ICachedPage page, boolean reinsert) {
CachedPage cPage = (CachedPage) page;
CacheBucket bucket = null;
- if(!page.confiscated()){
+ if (!page.confiscated()) {
return;
}
if (reinsert) {
int hash = hash(cPage.dpid);
bucket = pageMap[hash];
bucket.bucketLock.lock();
- if(DEBUG) {
+ if (DEBUG) {
confiscateLock.lock();
}
try {
@@ -1114,8 +1115,8 @@
cPage.next = bucket.cachedPage;
bucket.cachedPage = cPage;
cPage.pinCount.decrementAndGet();
- if(DEBUG){
- assert cPage.pinCount.get() == 0 ;
+ if (DEBUG) {
+ assert cPage.pinCount.get() == 0;
assert cPage.latch.getReadLockCount() == 0;
assert cPage.latch.getWriteHoldCount() == 0;
confiscatedPages.remove(cPage);
@@ -1123,22 +1124,22 @@
}
} finally {
bucket.bucketLock.unlock();
- if(DEBUG) {
+ if (DEBUG) {
confiscateLock.unlock();
}
}
} else {
cPage.invalidate();
cPage.pinCount.decrementAndGet();
- if(DEBUG){
+ if (DEBUG) {
assert cPage.pinCount.get() == 0;
assert cPage.latch.getReadLockCount() == 0;
assert cPage.latch.getWriteHoldCount() == 0;
confiscateLock.lock();
- try{
+ try {
confiscatedPages.remove(cPage);
confiscatedPagesOwner.remove(cPage);
- } finally{
+ } finally {
confiscateLock.unlock();
}
}
@@ -1186,14 +1187,14 @@
* _ONLY_ call this if you absolutely, positively know this file has no
dirty pages in the cache!
* Bypasses the normal lifecycle of a file handle and evicts all
references to it immediately.
*/
- public void purgeHandle(int fileId) throws HyracksDataException{
- synchronized(fileInfoMap){
- BufferedFileHandle fh = fileInfoMap.get(fileId);
- if(fh != null){
- ioManager.close(fh.getFileHandle());
- fileInfoMap.remove(fileId);
- fileMapManager.unregisterFile(fileId);
- }
+ public void purgeHandle(int fileId) throws HyracksDataException {
+ synchronized (fileInfoMap) {
+ BufferedFileHandle fh = fileInfoMap.get(fileId);
+ if (fh != null) {
+ ioManager.close(fh.getFileHandle());
+ fileInfoMap.remove(fileId);
+ fileMapManager.unregisterFile(fileId);
+ }
}
}
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/736
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ic77f4ee76a77f4b49cac6835bbb976eb9dc92d3a
Gerrit-PatchSet: 1
Gerrit-Project: hyracks
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <[email protected]>
