Repository: asterixdb Updated Branches: refs/heads/master aee552c08 -> 15e49e0db
Fix for ASTERIXDB-1725 This is tricky, the best method I could think to fix this involved simply putting the root page number in the metadata page rather than trying to calculate it somehow going backwards from the end of the file. I also fixed a bug discovered by Mike B where getLSNOffset was not taking into consideration page header sizes. Change-Id: I9f21e96ab045d331e4fc1c77b5c73b975e6260fa Reviewed-on: https://asterix-gerrit.ics.uci.edu/1331 Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Michael Blow <mb...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/15e49e0d Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/15e49e0d Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/15e49e0d Branch: refs/heads/master Commit: 15e49e0db5a40d68e6e6dbf60138d6ec61fd15b0 Parents: aee552c Author: Ian Maxon <ima...@apache.org> Authored: Thu Nov 3 14:42:45 2016 -0700 Committer: Ian Maxon <ima...@apache.org> Committed: Fri Nov 4 00:27:41 2016 -0700 ---------------------------------------------------------------------- .../am/common/api/IMetaDataPageManager.java | 4 ++ .../am/common/api/ITreeIndexMetaDataFrame.java | 6 ++- .../am/common/frames/LIFOMetaDataFrame.java | 14 +++++- .../freepage/LinkedMetaDataPageManager.java | 49 ++++++++++++++++++-- .../am/common/impls/AbstractTreeIndex.java | 22 +++------ .../freepage/VirtualMetaDataPageManager.java | 13 ++++++ .../impls/MultitenantVirtualBufferCache.java | 7 ++- .../am/lsm/common/impls/VirtualBufferCache.java | 12 +++-- .../storage/common/buffercache/BufferCache.java | 1 + .../common/buffercache/DebugBufferCache.java | 5 ++ .../common/buffercache/IBufferCache.java | 2 + 11 files changed, 109 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/15e49e0d/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IMetaDataPageManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IMetaDataPageManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IMetaDataPageManager.java index 3982a3a..5d87f99 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IMetaDataPageManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IMetaDataPageManager.java @@ -172,4 +172,8 @@ public interface IMetaDataPageManager { long getLSNOffset() throws HyracksDataException; public long getLastMarkerLSN() throws HyracksDataException; + + void setRootPage(int rootPage) throws HyracksDataException; + + int getRootPage() throws HyracksDataException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/15e49e0d/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexMetaDataFrame.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexMetaDataFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexMetaDataFrame.java index d4fbaa2..21e918d 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexMetaDataFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexMetaDataFrame.java @@ -25,7 +25,7 @@ public interface ITreeIndexMetaDataFrame { //Storage version #. Change this if you alter any tree frame formats to stop // possible corruption from old versions reading new formats. - public static final int VERSION = 2; + public static final int VERSION = 3; public void initBuffer(byte level); @@ -74,4 +74,8 @@ public interface ITreeIndexMetaDataFrame { public long getLastMarkerLSN(); public void setLastMarkerLSN(long lsn); + + void setRootPageNumber(int rootPage); + + int getRootPageNumber(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/15e49e0d/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java index d4194c2..5489dcc 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java @@ -49,7 +49,8 @@ public class LIFOMetaDataFrame implements ITreeIndexMetaDataFrame { public static final int LSN_OFFSET = ADDITIONAL_FILTERING_PAGE_OFFSET + 4; // 33 private static final int LAST_MARKER_LSN_OFFSET = LSN_OFFSET + 8; // 41 public static final int STORAGE_VERSION_OFFSET = LAST_MARKER_LSN_OFFSET + 4; //45 - private static final int HEADER_END_OFFSET = LAST_MARKER_LSN_OFFSET + 4; //49 + public static final int ROOT_PAGE_NUMBER = STORAGE_VERSION_OFFSET + 4; //49 + private static final int HEADER_END_OFFSET = ROOT_PAGE_NUMBER + 4; // 53 protected ICachedPage page = null; protected ByteBuffer buf = null; @@ -126,6 +127,7 @@ public class LIFOMetaDataFrame implements ITreeIndexMetaDataFrame { buf.putInt(NEXT_PAGE_OFFSET, -1); buf.putInt(ADDITIONAL_FILTERING_PAGE_OFFSET, -1); buf.putLong(LAST_MARKER_LSN_OFFSET, -1L); + buf.putInt(ROOT_PAGE_NUMBER, 0); buf.putInt(STORAGE_VERSION_OFFSET, VERSION); setValid(false); } @@ -192,4 +194,14 @@ public class LIFOMetaDataFrame implements ITreeIndexMetaDataFrame { public void setLSMComponentFilterPageId(int filterPage) { buf.putInt(ADDITIONAL_FILTERING_PAGE_OFFSET, filterPage); } + + @Override + public void setRootPageNumber(int rootPage) { + buf.putInt(ROOT_PAGE_NUMBER, rootPage); + } + + @Override + public int getRootPageNumber() { + return buf.getInt(ROOT_PAGE_NUMBER); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/15e49e0d/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedMetaDataPageManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedMetaDataPageManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedMetaDataPageManager.java index e7a1123..2b0566d 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedMetaDataPageManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedMetaDataPageManager.java @@ -18,7 +18,6 @@ */ package org.apache.hyracks.storage.am.common.freepage; - import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager; import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame; @@ -64,7 +63,7 @@ public class LinkedMetaDataPageManager implements IMetaDataPageManager { int newPage = metaFrame.getFreePage(); if (newPage < 0) { throw new HyracksDataException( - "Inconsistent Meta Page State. It has no space, but it also has no entries."); + "Inconsistent Meta Page State. It has no space, but it also has no entries."); } ICachedPage newNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, newPage), false); @@ -471,7 +470,7 @@ public class LinkedMetaDataPageManager implements IMetaDataPageManager { public long getLSNOffset() throws HyracksDataException { int metadataPageNum = getFirstMetadataPage(); if (metadataPageNum != IBufferCache.INVALID_PAGEID) { - return ((long)metadataPageNum * bufferCache.getPageSize()) + LIFOMetaDataFrame.LSN_OFFSET; + return ((long) metadataPageNum * bufferCache.getPageSizeWithHeader()) + LIFOMetaDataFrame.LSN_OFFSET; } return IMetaDataPageManager.INVALID_LSN_OFFSET; } @@ -496,4 +495,48 @@ public class LinkedMetaDataPageManager implements IMetaDataPageManager { } } } + + @Override + public void setRootPage(int rootPage) throws HyracksDataException { + ICachedPage metaNode; + if (!appendOnly) { + metaNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, getFirstMetadataPage()), false); + } else { + metaNode = confiscatedMetaNode; + } + ITreeIndexMetaDataFrame metaFrame = metaDataFrameFactory.createFrame(); + metaNode.acquireWriteLatch(); + try { + metaFrame.setPage(metaNode); + metaFrame.setRootPageNumber(rootPage); + } finally { + if (!appendOnly) { + metaNode.releaseWriteLatch(true); + bufferCache.unpin(metaNode); + } else { + metaNode.releaseWriteLatch(false); + } + } + } + + @Override + public int getRootPage() throws HyracksDataException { + ICachedPage metaNode; + if (!appendOnly || confiscatedMetaNode == null) { + metaNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, getFirstMetadataPage()), false); + } else { + metaNode = confiscatedMetaNode; + } + ITreeIndexMetaDataFrame metaFrame = metaDataFrameFactory.createFrame(); + metaNode.acquireReadLatch(); + try { + metaFrame.setPage(metaNode); + return metaFrame.getRootPageNumber(); + } finally { + metaNode.releaseReadLatch(); + if (!appendOnly || confiscatedMetaNode == null) { + bufferCache.unpin(metaNode); + } + } + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/15e49e0d/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java index 0dcfc90..5ac203e 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java @@ -114,7 +114,6 @@ public abstract class AbstractTreeIndex implements ITreeIndex { } freePageManager.open(fileId); - setRootAndMetadataPages(appendOnly); if (!appendOnly) { initEmptyTree(); freePageManager.close(); @@ -122,6 +121,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex { this.appendOnly = true; initCachedMetadataPage(); } + setRootPage(appendOnly); bufferCache.closeFile(fileId); } @@ -140,25 +140,14 @@ public abstract class AbstractTreeIndex implements ITreeIndex { } } - private void setRootAndMetadataPages(boolean appendOnly) throws HyracksDataException { + private void setRootPage(boolean appendOnly) throws HyracksDataException { if (!appendOnly) { // regular or empty tree rootPage = 1; bulkloadLeafStart = 2; } else { - //the root page is either page n-2 (no filter) or n-3 (filter) - int numPages = bufferCache.getNumPagesOfFile(fileId); - if (numPages > MINIMAL_TREE_PAGE_COUNT) { - int filterPageId = freePageManager.getFilterPageId(); - if (filterPageId > 0) { - rootPage = numPages - MINIMAL_TREE_PAGE_COUNT_WITH_FILTER; - } else { - rootPage = numPages - MINIMAL_TREE_PAGE_COUNT; - } - } else { - rootPage = 0; - } - + //root page is stored in MD page + rootPage = freePageManager.getRootPage(); //leaves start from the very beginning of the file. bulkloadLeafStart = 0; } @@ -201,7 +190,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex { } else { appendOnly = false; } - setRootAndMetadataPages(appendOnly); + setRootPage(appendOnly); // TODO: Should probably have some way to check that the tree is physically consistent // or that the file we just opened actually is a tree @@ -424,6 +413,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex { } } + freePageManager.setRootPage(rootPage); } protected void addLevel() throws HyracksDataException { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/15e49e0d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java index 158c68f..eab85d2 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java @@ -184,4 +184,17 @@ public class VirtualMetaDataPageManager implements IVirtualMetaDataPageManager { // Method doesn't make sense for this free page manager. return -1L; } + + @Override + public void setRootPage(int rootPage) throws HyracksDataException { + // This won't get called for an in-place index. The root page + // is maintained at a fixed location as in the below method. + } + + @Override + public int getRootPage() throws HyracksDataException { + // This also won't be called but the correct answer for an + // In-place index is always 1. + return 1; + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/15e49e0d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java index 748dbfe..3ef419e 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java @@ -91,6 +91,10 @@ public class MultitenantVirtualBufferCache implements IVirtualBufferCache { return vbc.getPageSize(); } + public int getPageSizeWithHeader() { + return vbc.getPageSizeWithHeader(); + } + @Override public int getNumPages() { return vbc.getNumPages(); @@ -104,7 +108,6 @@ public class MultitenantVirtualBufferCache implements IVirtualBufferCache { } } - @Override public synchronized void open() throws HyracksDataException { ++openCount; @@ -194,7 +197,7 @@ public class MultitenantVirtualBufferCache implements IVirtualBufferCache { } @Override - public int getFileReferenceCount(int fileId){ + public int getFileReferenceCount(int fileId) { return 0; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/15e49e0d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java index c8ce00f..abfc35c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java @@ -248,8 +248,8 @@ public class VirtualBufferCache implements IVirtualBufferCache { } else { largePages.getAndAdd(multiplier - origMultiplier); } - ((VirtualPage)cPage).buffer = newBuffer; - ((VirtualPage)cPage).multiplier = multiplier; + ((VirtualPage) cPage).buffer = newBuffer; + ((VirtualPage) cPage).multiplier = multiplier; } @Override @@ -270,6 +270,11 @@ public class VirtualBufferCache implements IVirtualBufferCache { } @Override + public int getPageSizeWithHeader() { + return pageSize; + } + + @Override public int getNumPages() { return numPages; } @@ -335,7 +340,7 @@ public class VirtualBufferCache implements IVirtualBufferCache { @Override public boolean isFull() { - return (nextFree + largePages.get()) >= numPages; + return (nextFree + largePages.get()) >= numPages; } private static class CacheBucket { @@ -390,6 +395,7 @@ public class VirtualBufferCache implements IVirtualBufferCache { public void releaseWriteLatch(boolean markDirty) { latch.writeLock().unlock(); } + public boolean confiscated() { return false; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/15e49e0d/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java index 3be1c46..ed33cc4 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java @@ -132,6 +132,7 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { return pageSize; } + @Override public int getPageSizeWithHeader() { return pageSize + RESERVED_HEADER_BYTES; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/15e49e0d/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java index 4a41ec0..6c88275 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java @@ -99,6 +99,11 @@ public class DebugBufferCache implements IBufferCache { } @Override + public int getPageSizeWithHeader() { + return bufferCache.getPageSizeWithHeader(); + } + + @Override public int getNumPages() { return bufferCache.getNumPages(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/15e49e0d/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java index ca87673..27e7982 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java @@ -62,6 +62,8 @@ public interface IBufferCache { int getPageSize(); + int getPageSizeWithHeader(); + public int getNumPages(); public int getNumPagesOfFile(int fileId) throws HyracksDataException;
