Michael Blow has submitted this change and it was merged. Change subject: ASTERIXDB-1474: Cannot query dataset with big obj ......................................................................
ASTERIXDB-1474: Cannot query dataset with big obj Fixes ASTERIXDB-1474: Cannot query a dataset with big object after restart - large page information managed by buffer cache, outside of page - size header written out a prefix to pages Change-Id: I2f47c551a053e193519a4337c338934f3e171afd Reviewed-on: https://asterix-gerrit.ics.uci.edu/920 Tested-by: Jenkins <[email protected]> Reviewed-by: abdullah alamoudi <[email protected]> --- M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql C asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql C asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql R asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql A asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh M asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java D hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java C hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IExtraPageBlockHelper.java M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java 54 files changed, 496 insertions(+), 447 deletions(-) Approvals: abdullah alamoudi: Looks good to me, approved Jenkins: Verified diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java index 7d92ce2..9fb4e94 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.common.config; +import org.apache.hyracks.storage.common.buffercache.IBufferCache; + public class AsterixStorageProperties extends AbstractAsterixProperties { private static final String STORAGE_BUFFERCACHE_PAGESIZE_KEY = "storage.buffercache.pagesize"; @@ -62,7 +64,7 @@ } public int getBufferCacheNumPages() { - return (int) (getBufferCacheSize() / getBufferCachePageSize()); + return (int) (getBufferCacheSize() / (getBufferCachePageSize() + IBufferCache.RESERVED_HEADER_BYTES)); } public int getBufferCacheMaxOpenFiles() { diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql new file mode 100644 index 0000000..cc46136 --- /dev/null +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql @@ -0,0 +1 @@ +create_and_start.sh diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql similarity index 66% copy from hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java copy to asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql index 22bec97..77c8179 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql @@ -16,10 +16,21 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.hyracks.storage.common.buffercache; +/* + * Test case Name : big_object_bulkload.aql + * Description : bulkload insert of large objects + * Expected Result : Success + * Date : 20th April 2016 + */ -public interface ILargePageHelper { +drop dataverse testdv2 if exists; +create dataverse testdv2; +use dataverse testdv2; - int getSupplementalBlockNumPages(CachedPage cPage); - int getSupplementalBlockPageId(CachedPage cPage); +create type testtype as closed { + id: int64, + name: string, + hobbies: {{string}} } + +create dataset testds(testtype) primary key id; diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql similarity index 74% copy from hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java copy to asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql index 22bec97..f1b3819 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql @@ -16,10 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.hyracks.storage.common.buffercache; +/** + * + * Big object (20 MB) loading test + * Expected result: success + * + */ -public interface ILargePageHelper { +use dataverse testdv2; - int getSupplementalBlockNumPages(CachedPage cPage); - int getSupplementalBlockPageId(CachedPage cPage); -} +load dataset testds +using localfs +(("path"="asterix_nc1://../../../../../../asterix-app/target/data/big-object/big_object_20M.adm"),("format"="adm")) pre-sorted; diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql new file mode 100644 index 0000000..3ba1dc0 --- /dev/null +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql @@ -0,0 +1 @@ +stop_and_start.sh diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql similarity index 79% rename from hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java rename to asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql index 22bec97..23c1bed 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql @@ -16,10 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.hyracks.storage.common.buffercache; +use dataverse testdv2; -public interface ILargePageHelper { - - int getSupplementalBlockNumPages(CachedPage cPage); - int getSupplementalBlockPageId(CachedPage cPage); -} +for $d in dataset("testds") +where $d.id = 1 +return $d diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql new file mode 100644 index 0000000..10e1a51 --- /dev/null +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql @@ -0,0 +1 @@ +stop_and_delete.sh diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template b/asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template new file mode 100644 index 0000000..5acf078 --- /dev/null +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template @@ -0,0 +1 @@ +{ "id": 1, "name": "Person One", "hobbies": {{ "%lorembytes:20971520%" }} } \ No newline at end of file diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh new file mode 100755 index 0000000..37bc7b0 --- /dev/null +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh @@ -0,0 +1,2 @@ +jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9; +$MANAGIX_HOME/bin/managix create -n asterix -c $MANAGIX_HOME/clusters/local/local.xml; diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh new file mode 100755 index 0000000..5f247db --- /dev/null +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh @@ -0,0 +1,3 @@ +$MANAGIX_HOME/bin/managix stop -n asterix; +$MANAGIX_HOME/bin/managix delete -n asterix; + diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh new file mode 100755 index 0000000..e2b0583 --- /dev/null +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh @@ -0,0 +1,2 @@ +$MANAGIX_HOME/bin/managix stop -n asterix; +$MANAGIX_HOME/bin/managix start -n asterix; diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml index 08b8e87..75b2457 100644 --- a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml @@ -28,6 +28,11 @@ <output-dir compare="Text">external_index</output-dir> </compilation-unit> </test-case> + <test-case FilePath="query_after_restart"> + <compilation-unit name="big_object_20M"> + <output-dir compare="Text">big_object_20M</output-dir> + </compilation-unit> + </test-case> </test-group> <test-group name="recover_after_abort"> diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java index 47eac7e..63709ed 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java @@ -43,6 +43,8 @@ public int syncWrite(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException; + public long syncWrite(IFileHandle fHandle, long offset, ByteBuffer[] dataArray) throws HyracksDataException; + public int syncRead(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException; public IIOFuture asyncWrite(IFileHandle fHandle, long offset, ByteBuffer data); diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java index 519a9d7..6588f3b 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java @@ -22,6 +22,7 @@ import java.io.FilenameFilter; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -49,7 +50,7 @@ public IOManager(List<IODeviceHandle> devices) throws HyracksException { this.ioDevices = Collections.unmodifiableList(devices); - workAreaIODevices = new ArrayList<IODeviceHandle>(); + workAreaIODevices = new ArrayList<>(); for (IODeviceHandle d : ioDevices) { if (d.getWorkAreaPath() != null) { new File(d.getPath(), d.getWorkAreaPath()).mkdirs(); @@ -109,13 +110,48 @@ } } + @Override + public long syncWrite(IFileHandle fHandle, long offset, ByteBuffer[] dataArray) throws HyracksDataException { + try { + if (fHandle == null) { + throw new IllegalStateException("Trying to write to a deleted file."); + } + int n = 0; + int remaining = 0; + for (ByteBuffer buf : dataArray) { + remaining += buf.remaining(); + } + final FileChannel fileChannel = ((FileHandle) fHandle).getFileChannel(); + while (remaining > 0) { + long len; + synchronized (fileChannel) { + fileChannel.position(offset); + len = fileChannel.write(dataArray); + } + if (len < 0) { + throw new HyracksDataException( + "Error writing to file: " + ((FileHandle) fHandle).getFileReference().toString()); + } + remaining -= len; + offset += len; + n += len; + } + return n; + } catch (HyracksDataException e) { + throw e; + } catch (IOException e) { + throw new HyracksDataException(e); + } + } + /** * Please do check the return value of this read! * * @param fHandle * @param offset * @param data - * @return The number of bytes read, possibly zero, or -1 if the given offset is greater than or equal to the file's current size + * @return The number of bytes read, possibly zero, or -1 if the given offset is greater than or equal to the file's + * current size * @throws HyracksDataException */ @Override diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java index 1081dc8..e0abb1c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java @@ -38,5 +38,7 @@ public void setLargeFlag(boolean largePage); + public boolean getLargeFlag(); + public void validate(PageValidationInfo pvi) throws HyracksDataException; } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java index 7648ee6..2e072ce 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java @@ -21,14 +21,13 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; -import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager; -import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference; import org.apache.hyracks.storage.am.common.api.TreeIndexException; import org.apache.hyracks.storage.am.common.ophelpers.FindTupleMode; import org.apache.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy; import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; public interface IBTreeLeafFrame extends IBTreeFrame { public int findTupleIndex(ITupleReference searchKey, ITreeIndexTupleReference pageTuple, MultiComparator cmp, @@ -53,12 +52,6 @@ public int getNextLeaf(); - public void configureLargePage(int supplementalPages, int supplementalBlockPageId); - - public boolean isLargePage(); - - public int getSupplementalNumPages(); - - void ensureCapacity(IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, - IBufferCache bufferCache, ITupleReference tuple) throws HyracksDataException; + void ensureCapacity(IBufferCache bufferCache, ITupleReference tuple, IExtraPageBlockHelper extraPageBlockHelper) + throws HyracksDataException; } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java index 701f44f..258a5d6 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java @@ -33,11 +33,9 @@ import org.apache.hyracks.storage.am.btree.impls.FieldPrefixPrefixTupleReference; import org.apache.hyracks.storage.am.btree.impls.FieldPrefixSlotManager; import org.apache.hyracks.storage.am.btree.impls.FieldPrefixTupleReference; -import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager; import org.apache.hyracks.storage.am.common.api.ISplitKey; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameCompressor; -import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; import org.apache.hyracks.storage.am.common.api.TreeIndexException; @@ -51,7 +49,7 @@ import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriter; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; +import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; /** * WARNING: only works when tupleWriter is an instance of TypeAwareTupleWriter @@ -537,23 +535,13 @@ } @Override - public void configureLargePage(int supplementalPages, int supplementalBlockPageId) { - throw new IllegalStateException(); - } - - @Override public void setLargeFlag(boolean largePage) { throw new IllegalStateException(); } @Override - public boolean isLargePage() { + public boolean getLargeFlag() { return false; - } - - @Override - public int getSupplementalNumPages() { - throw new IllegalStateException(); } public int getPrefixTupleCount() { @@ -596,7 +584,8 @@ } @Override - public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey, IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache) + public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey, + IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache) throws HyracksDataException { BTreeFieldPrefixNSMLeafFrame rf = (BTreeFieldPrefixNSMLeafFrame) rightFrame; @@ -804,12 +793,8 @@ } @Override - public ILargePageHelper getLargePageHelper() { - return null; - } - - @Override - public void ensureCapacity(IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache, ITupleReference tuple) throws HyracksDataException { + public void ensureCapacity(IBufferCache bufferCache, ITupleReference tuple, IExtraPageBlockHelper helper) + throws HyracksDataException { throw new IllegalStateException("nyi"); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java index c558d30..f7b270c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java @@ -22,7 +22,6 @@ import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; public class BTreeFieldPrefixNSMLeafFrameFactory implements ITreeIndexFrameFactory { @@ -44,8 +43,4 @@ return tupleWriterFactory; } - @Override - public ILargePageHelper getLargePageHelper() { - return null; - } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java deleted file mode 100644 index 79f7f53..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hyracks.storage.am.btree.frames; - -import java.nio.ByteBuffer; - -import org.apache.hyracks.storage.common.buffercache.CachedPage; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; - -public class BTreeLargeFrameHelper implements ILargePageHelper { - - public static final BTreeLargeFrameHelper INSTANCE = new BTreeLargeFrameHelper(); - - private BTreeLargeFrameHelper() { - } - - @Override - public int getSupplementalBlockNumPages(CachedPage cPage) { - ByteBuffer buffer = cPage.getBuffer(); - return BTreeNSMLeafFrame.isLargePage(buffer) ? BTreeNSMLeafFrame.getSupplementalNumPages(buffer) : 0; - } - - @Override - public int getSupplementalBlockPageId(CachedPage cPage) { - ByteBuffer buffer = cPage.getBuffer(); - return BTreeNSMLeafFrame.isLargePage(buffer) ? BTreeNSMLeafFrame.getSupplementalPageId(buffer) : -1; - } -} diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java index c2b18b7..9505f8a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java @@ -29,10 +29,8 @@ import org.apache.hyracks.storage.am.btree.api.IBTreeInteriorFrame; import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext.PageValidationInfo; import org.apache.hyracks.storage.am.btree.impls.RangePredicate; -import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager; import org.apache.hyracks.storage.am.common.api.ISplitKey; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; -import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; import org.apache.hyracks.storage.am.common.api.TreeIndexException; @@ -43,6 +41,7 @@ import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; import org.apache.hyracks.storage.am.common.ophelpers.SlotOffTupleOff; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; public class BTreeNSMInteriorFrame extends TreeIndexNSMFrame implements IBTreeInteriorFrame { @@ -55,7 +54,7 @@ private MultiComparator cmp; public BTreeNSMInteriorFrame(ITreeIndexTupleWriter tupleWriter) { - super(tupleWriter, new OrderedSlotManager(), null); + super(tupleWriter, new OrderedSlotManager()); cmpFrameTuple = tupleWriter.createTupleReference(); previousFt = tupleWriter.createTupleReference(); } @@ -201,7 +200,7 @@ @Override public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey, - IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache) + IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache) throws HyracksDataException, TreeIndexException { ByteBuffer right = rightFrame.getBuffer(); int tupleCount = getTupleCount(); diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java index 3737486..4ecb83d 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java @@ -22,7 +22,6 @@ import org.apache.hyracks.storage.am.btree.api.IBTreeInteriorFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; public class BTreeNSMInteriorFrameFactory implements ITreeIndexFrameFactory { @@ -44,8 +43,4 @@ return tupleWriterFactory; } - @Override - public ILargePageHelper getLargePageHelper() { - return null; - } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java index 8f560fe..406b494 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java @@ -25,10 +25,8 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame; import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext.PageValidationInfo; -import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager; import org.apache.hyracks.storage.am.common.api.ISplitKey; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; -import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; import org.apache.hyracks.storage.am.common.api.TreeIndexException; @@ -40,25 +38,23 @@ import org.apache.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy; import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; +import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFrame { protected static final int nextLeafOff = flagOff + 1; // 22 - protected static final int supplementalNumPagesOff = nextLeafOff + 4; // 26 - protected static final int supplementalPageIdOff = supplementalNumPagesOff + 4; // 30 private MultiComparator cmp; private final ITreeIndexTupleReference previousFt; - public BTreeNSMLeafFrame(ITreeIndexTupleWriter tupleWriter, ILargePageHelper largePageHelper) { - super(tupleWriter, new OrderedSlotManager(), largePageHelper); + public BTreeNSMLeafFrame(ITreeIndexTupleWriter tupleWriter) { + super(tupleWriter, new OrderedSlotManager()); previousFt = tupleWriter.createTupleReference(); } @Override public int getPageHeaderSize() { - return supplementalPageIdOff + 4; + return nextLeafOff + 4; } @Override @@ -70,8 +66,6 @@ public void initBuffer(byte level) { super.initBuffer(level); buf.putInt(nextLeafOff, -1); - buf.putInt(supplementalNumPagesOff, 0); - buf.putInt(supplementalPageIdOff, -1); } @Override @@ -82,28 +76,6 @@ @Override public int getNextLeaf() { return buf.getInt(nextLeafOff); - } - - public static int getSupplementalNumPages(ByteBuffer buf) { - return buf.getInt(supplementalNumPagesOff); - } - - public int getSupplementalNumPages() { - return getSupplementalNumPages(buf); - } - - public static int getSupplementalPageId(ByteBuffer buf) { - return buf.getInt(supplementalPageIdOff); - } - - public int getSupplementalPageId() { - return getSupplementalPageId(buf); - } - - public void configureLargePage(int supplementalPages, int supplementalBlockPageId) { - setLargeFlag(true); - buf.putInt(supplementalNumPagesOff, supplementalPages); - buf.putInt(supplementalPageIdOff, supplementalBlockPageId); } @Override @@ -204,7 +176,7 @@ boolean isLargeTuple(int tupleSize) { // TODO(mblow): make page size available to avoid calculating it - int pageSize = isLargePage() ? buf.capacity() / (getSupplementalNumPages() + 1) : buf.capacity(); + int pageSize = getLargeFlag() ? buf.capacity() / page.getFrameSizeMultiplier() : buf.capacity(); return tupleSize > getMaxTupleSize(pageSize); } @@ -228,7 +200,7 @@ int oldTupleBytes = frameTuple.getTupleSize(); int newTupleBytes = tupleWriter.bytesRequired(newTuple); FrameOpSpaceStatus status = hasSpaceUpdate(oldTupleBytes, newTupleBytes); - if (status == FrameOpSpaceStatus.INSUFFICIENT_SPACE && (isLargePage() || getTupleCount() == 1) + if (status == FrameOpSpaceStatus.INSUFFICIENT_SPACE && (getLargeFlag() || getTupleCount() == 1) && isLargeTuple(newTupleBytes)) { return FrameOpSpaceStatus.EXPAND; } @@ -237,8 +209,8 @@ @Override public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey, - IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache) - throws HyracksDataException { + IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache) + throws HyracksDataException { int tupleSize = getBytesRequiredToWriteTuple(tuple); @@ -249,7 +221,7 @@ // Find split point, and determine into which frame the new tuple should // be inserted into. - BTreeNSMLeafFrame targetFrame = null; + BTreeNSMLeafFrame targetFrame; frameTuple.resetByTupleIndex(this, tupleCount - 1); if (cmp.compare(tuple, frameTuple) > 0) { // This is a special optimization case when the tuple to be inserted is the largest key on the page. @@ -275,22 +247,14 @@ targetFrame = this; } int tuplesToRight = tupleCount - tuplesToLeft; - int supplementalPages = 0; - int supplementalPageId = -1; - if (isLargePage()) { - ((BTreeNSMLeafFrame) rightFrame).growCapacity(freePageManager, metaFrame, bufferCache, + if (getLargeFlag()) { + ((BTreeNSMLeafFrame) rightFrame).growCapacity(extraPageBlockHelper, bufferCache, buf.capacity() - rightFrame.getBuffer().capacity()); - supplementalPages = ((BTreeNSMLeafFrame) rightFrame).getSupplementalNumPages(); - supplementalPageId = ((BTreeNSMLeafFrame) rightFrame).getSupplementalPageId(); } ByteBuffer right = rightFrame.getBuffer(); // Copy entire page. System.arraycopy(buf.array(), 0, right.array(), 0, buf.capacity()); - if (isLargePage()) { - // restore the supplemental page metadata - ((BTreeNSMLeafFrame) rightFrame).configureLargePage(supplementalPages, supplementalPageId); - } // On the right page we need to copy rightmost slots to the left. int src = rightFrame.getSlotManager().getSlotEndOff(); @@ -309,7 +273,7 @@ } if (tupleLarge) { - targetFrame.ensureCapacity(freePageManager, metaFrame, bufferCache, tuple); + targetFrame.ensureCapacity(bufferCache, tuple, extraPageBlockHelper); } // Insert the new tuple. @@ -317,7 +281,7 @@ // it's safe to catch this exception since it will have been caught // before reaching here try { - targetTupleIndex = ((BTreeNSMLeafFrame) targetFrame).findInsertTupleIndex(tuple); + targetTupleIndex = targetFrame.findInsertTupleIndex(tuple); } catch (TreeIndexException e) { throw new IllegalStateException(e); } @@ -332,13 +296,13 @@ splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer(), 0); } - public void ensureCapacity(IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, - IBufferCache bufferCache, ITupleReference tuple) throws HyracksDataException { + public void ensureCapacity(IBufferCache bufferCache, ITupleReference tuple, + IExtraPageBlockHelper extraPageBlockHelper) throws HyracksDataException { int gapBytes = getBytesRequiredToWriteTuple(tuple) - getFreeContiguousSpace(); - growCapacity(freePageManager, metaFrame, bufferCache, gapBytes); + growCapacity(extraPageBlockHelper, bufferCache, gapBytes); } - public void growCapacity(IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, + public void growCapacity(IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache, int delta) throws HyracksDataException { if (delta <= 0) { setLargeFlag(true); @@ -346,35 +310,21 @@ } int deltaPages = (int) Math.ceil((double) delta / bufferCache.getPageSize()); int framePagesOld = getBuffer().capacity() / bufferCache.getPageSize(); - int oldSupplementalPages = 0; - int oldSupplementalPageId = -1; - if (isLargePage()) { - oldSupplementalPages = getSupplementalNumPages(); - oldSupplementalPageId = getSupplementalPageId(); - } - - configureLargePage(framePagesOld + deltaPages - 1, - freePageManager.getFreePageBlock(metaFrame, framePagesOld + deltaPages - 1)); - - int pageDelta = (framePagesOld + deltaPages) - 1 - oldSupplementalPages; + int newMultiplier = framePagesOld + deltaPages; // we need to get the old slot offsets before we grow int oldSlotEnd = slotManager.getSlotEndOff(); int oldSlotStart = slotManager.getSlotStartOff() + slotManager.getSlotSize(); - bufferCache.resizePage(getPage(), framePagesOld + deltaPages); - buf = getPage().getBuffer(); + bufferCache.resizePage(getPage(), newMultiplier, extraPageBlockHelper); - // return the dropped supplemental pages to the page manager... - if (oldSupplementalPages > 0) { - freePageManager.addFreePageBlock(metaFrame, oldSupplementalPageId, oldSupplementalPages); - } + buf = getPage().getBuffer(); // fixup the slots System.arraycopy(buf.array(), oldSlotEnd, buf.array(), slotManager.getSlotEndOff(), oldSlotStart - oldSlotEnd); // fixup total free space counter - buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + (bufferCache.getPageSize() * pageDelta)); + buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + (bufferCache.getPageSize() * deltaPages)); } @Override @@ -416,8 +366,6 @@ public String printHeader() { StringBuilder strBuilder = new StringBuilder(super.printHeader()); strBuilder.append("nextLeafOff: " + nextLeafOff + "\n"); - strBuilder.append("supplementalNumPagesOff: " + supplementalNumPagesOff + "\n"); - strBuilder.append("supplementalPageIdOff: " + supplementalPageIdOff + "\n"); return strBuilder.toString(); } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java index 2b7f12b..3cf0ba5 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java @@ -22,7 +22,6 @@ import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; public class BTreeNSMLeafFrameFactory implements ITreeIndexFrameFactory { @@ -36,7 +35,7 @@ @Override public IBTreeLeafFrame createFrame() { - return new BTreeNSMLeafFrame(tupleWriterFactory.createTupleWriter(), getLargePageHelper()); + return new BTreeNSMLeafFrame(tupleWriterFactory.createTupleWriter()); } @Override @@ -44,8 +43,4 @@ return tupleWriterFactory; } - @Override - public ILargePageHelper getLargePageHelper() { - return BTreeLargeFrameHelper.INSTANCE; - } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java index 3043940..5995004 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java @@ -66,7 +66,6 @@ import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; import org.apache.hyracks.storage.common.buffercache.BufferCache; -import org.apache.hyracks.storage.common.buffercache.CachedPage; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.file.BufferedFileHandle; @@ -103,8 +102,7 @@ RangePredicate diskOrderScanPred = new RangePredicate(null, null, true, true, ctx.cmp, ctx.cmp); int maxPageId = freePageManager.getMaxPage(ctx.metaFrame); int currentPageId = bulkloadLeafStart; - ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false, - largePageHelper); + ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false); page.acquireReadLatch(); try { cursor.setBufferCache(bufferCache); @@ -136,7 +134,7 @@ } private void validate(BTreeOpContext ctx, int pageId) throws HyracksDataException { - ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper); + ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false); ctx.interiorFrame.setPage(page); PageValidationInfo currentPvi = ctx.validationInfos.peekFirst(); @@ -220,8 +218,7 @@ ICachedPage originalPage = ctx.interiorFrame.getPage(); for (int i = 0; i < ctx.smPages.size(); i++) { int pageId = ctx.smPages.get(i); - ICachedPage smPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, - largePageHelper); + ICachedPage smPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false); smPage.acquireWriteLatch(); try { ctx.interiorFrame.setPage(smPage); @@ -246,17 +243,17 @@ private void createNewRoot(BTreeOpContext ctx) throws HyracksDataException, TreeIndexException { // Make sure the root is always in the same page. ICachedPage leftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, ctx.splitKey.getLeftPage()), - false, largePageHelper); + false); leftNode.acquireWriteLatch(); try { int newLeftId = freePageManager.getFreePage(ctx.metaFrame); - ICachedPage newLeftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, newLeftId), true, - largePageHelper); + ICachedPage newLeftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, newLeftId), true); newLeftNode.acquireWriteLatch(); try { boolean largePage = false; if (leftNode.getBuffer().capacity() > newLeftNode.getBuffer().capacity()) { - bufferCache.resizePage(newLeftNode, leftNode.getBuffer().capacity() / bufferCache.getPageSize()); + bufferCache.resizePage(newLeftNode, leftNode.getBuffer().capacity() / bufferCache.getPageSize(), + ctx); largePage = true; } // Copy left child to new left child. @@ -268,7 +265,7 @@ long leftNodeLSN = ctx.interiorFrame.getPageLsn(); // Initialize new root (leftNode becomes new root). if (largePage) { - bufferCache.resizePage(leftNode, 1); + bufferCache.resizePage(leftNode, 1, ctx); ctx.interiorFrame.setPage(leftNode); ctx.interiorFrame.setLargeFlag(false); } else { @@ -372,7 +369,7 @@ switch (spaceStatus) { case EXPAND: { // TODO: avoid repeated calculation of tuple size - ctx.leafFrame.ensureCapacity(freePageManager, ctx.metaFrame, bufferCache, tuple); + ctx.leafFrame.ensureCapacity(bufferCache, tuple, ctx); } // fall-through case SUFFICIENT_CONTIGUOUS_SPACE: { @@ -432,8 +429,7 @@ } } int rightPageId = freePageManager.getFreePage(ctx.metaFrame); - ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true, - largePageHelper); + ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true); rightNode.acquireWriteLatch(); try { IBTreeLeafFrame rightFrame = ctx.createLeafFrame(); @@ -449,7 +445,7 @@ } else { ctx.modificationCallback.found(null, tuple); } - ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey, freePageManager, ctx.metaFrame, bufferCache); + ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey, ctx, bufferCache); ctx.smPages.add(pageId); ctx.smPages.add(rightPageId); @@ -492,7 +488,7 @@ ctx.modificationCallback.found(beforeTuple, tuple); ctx.leafFrame.delete(tuple, oldTupleIndex); ctx.leafFrame.compact(); - ctx.leafFrame.ensureCapacity(freePageManager, ctx.metaFrame, bufferCache, tuple); + ctx.leafFrame.ensureCapacity(bufferCache, tuple, ctx); int targetTupleIndex = ctx.leafFrame.findInsertTupleIndex(tuple); ctx.leafFrame.insert(tuple, targetTupleIndex); ctx.splitKey.reset(); @@ -550,8 +546,7 @@ switch (spaceStatus) { case INSUFFICIENT_SPACE: { int rightPageId = freePageManager.getFreePage(ctx.metaFrame); - ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true, - largePageHelper); + ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true); rightNode.acquireWriteLatch(); try { IBTreeFrame rightFrame = ctx.createInteriorFrame(); @@ -560,8 +555,7 @@ rightFrame.setMultiComparator(ctx.cmp); // instead of creating a new split key, use the existing // splitKey - ctx.interiorFrame.split(rightFrame, ctx.splitKey.getTuple(), ctx.splitKey, freePageManager, - ctx.metaFrame, bufferCache); + ctx.interiorFrame.split(rightFrame, ctx.splitKey.getTuple(), ctx.splitKey, ctx, bufferCache); ctx.smPages.add(pageId); ctx.smPages.add(rightPageId); ctx.interiorFrame.setSmFlag(true); @@ -631,7 +625,7 @@ } private ICachedPage isConsistent(int pageId, BTreeOpContext ctx) throws Exception { - ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper); + ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false); node.acquireReadLatch(); ctx.interiorFrame.setPage(node); boolean isConsistent = ctx.pageLsns.getLast() == ctx.interiorFrame.getPageLsn(); @@ -645,7 +639,7 @@ private void performOp(int pageId, ICachedPage parent, boolean parentIsReadLatched, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException { - ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper); + ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false); ctx.interiorFrame.setPage(node); // this check performs an unprotected read in the page // the following could happen: TODO fill out @@ -707,7 +701,7 @@ // Is there a propagated split key? if (ctx.splitKey.getBuffer() != null) { ICachedPage interiorNode = bufferCache.pin( - BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper); + BufferedFileHandle.getDiskPageId(fileId, pageId), false); interiorNode.acquireWriteLatch(); try { // Insert or update op. Both can cause split keys to propagate upwards. @@ -839,8 +833,7 @@ private BTreeOpContext createOpContext(IIndexAccessor accessor, IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) { return new BTreeOpContext(accessor, leafFrameFactory, interiorFrameFactory, - freePageManager.getMetaDataFrameFactory().createFrame(), cmpFactories, modificationCallback, - searchCallback); + freePageManager, cmpFactories, modificationCallback, searchCallback); } @SuppressWarnings("rawtypes") @@ -857,7 +850,7 @@ public void printTree(int pageId, ICachedPage parent, boolean unpin, IBTreeLeafFrame leafFrame, IBTreeInteriorFrame interiorFrame, byte treeHeight, ISerializerDeserializer[] keySerdes, StringBuilder strBuilder, MultiComparator cmp) throws Exception { - ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper); + ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false); node.acquireReadLatch(); try { if (parent != null && unpin == true) { @@ -1077,21 +1070,18 @@ int headerSize = Math.max(leafFrame.getPageHeaderSize(), interiorFrame.getPageHeaderSize()); final int multiplier = (int) Math .ceil((double) tupleSize / (bufferCache.getPageSize() - headerSize)); - leafFrontier.page = bufferCache.confiscateLargePage(dpid, multiplier); - ((CachedPage) leafFrontier.page).setLargePageHelper(largePageHelper); + if (multiplier > 1) { + leafFrontier.page = bufferCache.confiscateLargePage(dpid, multiplier, + freePageManager.getFreePageBlock(metaFrame, multiplier - 1)); + } else { + leafFrontier.page = bufferCache.confiscatePage(dpid); + } leafFrame.setPage(leafFrontier.page); leafFrame.initBuffer((byte) 0); - if (multiplier > 1) { - int supplementalPages = multiplier - 1; - ((IBTreeLeafFrame) leafFrame).configureLargePage(supplementalPages, - freePageManager.getFreePageBlock(metaFrame, supplementalPages)); - } else { - ((IBTreeLeafFrame) leafFrame).setLargeFlag(true); - } + ((IBTreeLeafFrame) leafFrame).setLargeFlag(true); } else { final long dpid = BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId); leafFrontier.page = bufferCache.confiscatePage(dpid); - ((CachedPage) leafFrontier.page).setLargePageHelper(largePageHelper); leafFrame.setPage(leafFrontier.page); leafFrame.initBuffer((byte) 0); } @@ -1161,7 +1151,6 @@ propagateBulk(level + 1, pagesToWrite); frontier.page = bufferCache.confiscatePage(BufferCache.INVALID_DPID); - ((CachedPage) frontier.page).setLargePageHelper(largePageHelper); interiorFrame.setPage(frontier.page); interiorFrame.initBuffer((byte) level); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java index 974860e..2297cc5 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java @@ -25,7 +25,6 @@ import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame; -import org.apache.hyracks.storage.am.btree.frames.BTreeLargeFrameHelper; import org.apache.hyracks.storage.am.common.api.ICursorInitialState; import org.apache.hyracks.storage.am.common.api.ISearchPredicate; import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor; @@ -119,8 +118,7 @@ private void fetchNextLeafPage(int nextLeafPage) throws HyracksDataException { do { - ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false, - BTreeLargeFrameHelper.INSTANCE); + ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false); if (exclusiveLatchNodes) { nextLeaf.acquireWriteLatch(); page.releaseWriteLatch(isPageDirty); diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java index 8513368..3649433 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java @@ -33,6 +33,7 @@ import org.apache.hyracks.storage.am.btree.api.ITupleAcceptor; import org.apache.hyracks.storage.am.common.api.IIndexAccessor; import org.apache.hyracks.storage.am.common.api.IIndexOperationContext; +import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager; import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback; import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback; import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor; @@ -43,8 +44,9 @@ import org.apache.hyracks.storage.am.common.ophelpers.IntArrayList; import org.apache.hyracks.storage.am.common.ophelpers.LongArrayList; import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; +import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; -public class BTreeOpContext implements IIndexOperationContext { +public class BTreeOpContext implements IIndexOperationContext, IExtraPageBlockHelper { private final int INIT_ARRAYLIST_SIZE = 6; public IIndexAccessor accessor; @@ -53,7 +55,8 @@ public ITreeIndexFrameFactory interiorFrameFactory; public IBTreeLeafFrame leafFrame; public IBTreeInteriorFrame interiorFrame; - public ITreeIndexMetaDataFrame metaFrame; + public final IMetaDataPageManager freePageManager; + public final ITreeIndexMetaDataFrame metaFrame; public IndexOperation op; public ITreeIndexCursor cursor; public BTreeCursorInitialState cursorInitialState; @@ -75,9 +78,9 @@ public final ITreeIndexTupleReference leafFrameTuple; public BTreeOpContext(IIndexAccessor accessor, ITreeIndexFrameFactory leafFrameFactory, - ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexMetaDataFrame metaFrame, - IBinaryComparatorFactory[] cmpFactories, IModificationOperationCallback modificationCallback, - ISearchOperationCallback searchCallback) { + ITreeIndexFrameFactory interiorFrameFactory, IMetaDataPageManager freePageManager, + IBinaryComparatorFactory[] cmpFactories, IModificationOperationCallback modificationCallback, + ISearchOperationCallback searchCallback) { this.accessor = accessor; if (cmpFactories[0] != null) { @@ -97,7 +100,8 @@ if (interiorFrame != null && this.cmp != null) { interiorFrame.setMultiComparator(cmp); } - this.metaFrame = metaFrame; + this.freePageManager = freePageManager; + this.metaFrame = freePageManager.getMetaDataFrameFactory().createFrame(); this.pageLsns = new LongArrayList(INIT_ARRAYLIST_SIZE, INIT_ARRAYLIST_SIZE); this.smoCount = 0; this.modificationCallback = modificationCallback; @@ -233,4 +237,14 @@ this.modificationCallback = modificationCallback; this.searchCallback = searchCallback; } + + @Override + public int getFreeBlock(int size) throws HyracksDataException { + return freePageManager.getFreePageBlock(metaFrame, size); + } + + @Override + public void returnFreePageBlock(int blockPageId, int size) throws HyracksDataException { + freePageManager.addFreePageBlock(metaFrame, blockPageId, size); + } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java index 975bd9b..3301e37 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java @@ -25,7 +25,6 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.dataflow.common.util.TupleUtils; import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame; -import org.apache.hyracks.storage.am.btree.frames.BTreeLargeFrameHelper; import org.apache.hyracks.storage.am.common.api.ICursorInitialState; import org.apache.hyracks.storage.am.common.api.IIndexAccessor; import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback; @@ -118,8 +117,7 @@ private void fetchNextLeafPage(int nextLeafPage) throws HyracksDataException { do { - ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false, - BTreeLargeFrameHelper.INSTANCE); + ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false); if (exclusiveLatchNodes) { nextLeaf.acquireWriteLatch(); page.releaseWriteLatch(isPageDirty); diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java index 5b6cee9..7c2abb1 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java @@ -38,7 +38,6 @@ import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; import org.apache.hyracks.storage.common.file.IFileMapProvider; public class BTreeUtils { @@ -49,7 +48,6 @@ ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType); ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory); ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory(); - ILargePageHelper largePageHelper = leafFrameFactory.getLargePageHelper(); IMetaDataPageManager freePageManager; freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory); BTree btree = new BTree(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory, @@ -63,7 +61,6 @@ TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits); ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType); ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory); - ILargePageHelper largePageHelper = leafFrameFactory.getLargePageHelper(); BTree btree = new BTree(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, typeTraits.length, file); return btree; diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java index 711db9e..037c183 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java @@ -27,7 +27,7 @@ import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; +import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; public interface ITreeIndexFrame { @@ -73,7 +73,7 @@ public String printHeader(); public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey, - IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache) + IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache) throws HyracksDataException, TreeIndexException; public ISlotManager getSlotManager(); @@ -104,6 +104,4 @@ public ITreeIndexTupleReference createTupleReference(); public void setMultiComparator(MultiComparator cmp); - - ILargePageHelper getLargePageHelper(); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java index 248baf5..8a705dc 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java @@ -20,10 +20,7 @@ import java.io.Serializable; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; - public interface ITreeIndexFrameFactory extends Serializable { ITreeIndexFrame createFrame(); ITreeIndexTupleWriterFactory getTupleWriterFactory(); - ILargePageHelper getLargePageHelper(); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java index ea6bcac..72c6fe2 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java @@ -31,7 +31,6 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; import org.apache.hyracks.storage.am.common.ophelpers.SlotOffTupleOff; import org.apache.hyracks.storage.common.buffercache.ICachedPage; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; public abstract class TreeIndexNSMFrame implements ITreeIndexFrame { @@ -51,14 +50,12 @@ protected ITreeIndexTupleWriter tupleWriter; protected ITreeIndexTupleReference frameTuple; - protected ILargePageHelper largePageHelper; - public TreeIndexNSMFrame(ITreeIndexTupleWriter tupleWriter, ISlotManager slotManager, ILargePageHelper largePageHelper) { + public TreeIndexNSMFrame(ITreeIndexTupleWriter tupleWriter, ISlotManager slotManager) { this.tupleWriter = tupleWriter; this.frameTuple = tupleWriter.createTupleReference(); this.slotManager = slotManager; this.slotManager.setFrame(this); - this.largePageHelper = largePageHelper; } @Override @@ -101,12 +98,8 @@ } } - public static boolean isLargePage(ByteBuffer buf) { + public boolean getLargeFlag() { return (buf.get(flagOff) & largeFlagBit) != 0; - } - - public boolean isLargePage() { - return isLargePage(buf); } @Override @@ -342,8 +335,4 @@ return buf.capacity() - getFreeSpaceOff() - (getTupleCount() * slotManager.getSlotSize()); } - @Override - public ILargePageHelper getLargePageHelper() { - return largePageHelper; - } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java index b7b39d4..3a7eb77 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java @@ -26,12 +26,20 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; -import org.apache.hyracks.storage.am.common.api.*; +import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader; +import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager; +import org.apache.hyracks.storage.am.common.api.ITreeIndex; +import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor; +import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; +import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; +import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame; +import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; +import org.apache.hyracks.storage.am.common.api.IndexException; +import org.apache.hyracks.storage.am.common.api.TreeIndexException; import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; import org.apache.hyracks.storage.common.file.BufferedFileHandle; import org.apache.hyracks.storage.common.file.IFileMapProvider; @@ -62,8 +70,6 @@ protected int bulkloadLeafStart = 0; - protected final ILargePageHelper largePageHelper; - public AbstractTreeIndex(IBufferCache bufferCache, IFileMapProvider fileMapProvider, IMetaDataPageManager freePageManager, ITreeIndexFrameFactory interiorFrameFactory, @@ -78,7 +84,6 @@ this.cmpFactories = cmpFactories; this.fieldCount = fieldCount; this.file = file; - this.largePageHelper = leafFrameFactory.getLargePageHelper(); } public synchronized void create() throws HyracksDataException { @@ -125,8 +130,7 @@ ITreeIndexFrame frame = leafFrameFactory.createFrame(); ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame(); freePageManager.init(metaFrame, rootPage); - ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true, - largePageHelper); + ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true); rootNode.acquireWriteLatch(); try { frame.setPage(rootNode); @@ -244,8 +248,7 @@ if(freePageManager.appendOnlyMode() && bufferCache.getNumPagesOfFile(fileId) <= MINIMAL_TREE_PAGE_COUNT){ return true; } - ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false, - largePageHelper); + ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false); rootNode.acquireReadLatch(); try { frame.setPage(rootNode); @@ -263,8 +266,7 @@ public byte getTreeHeight(ITreeIndexFrame frame) throws HyracksDataException { - ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false, - largePageHelper); + ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false); rootNode.acquireReadLatch(); try { frame.setPage(rootNode); @@ -394,13 +396,12 @@ //move the root page to the first data page if necessary bufferCache.finishQueue(); if (!appendOnly) { - ICachedPage newRoot = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true, - largePageHelper); + ICachedPage newRoot = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true); newRoot.acquireWriteLatch(); //root will be the highest frontier NodeFrontier lastNodeFrontier = nodeFrontiers.get(nodeFrontiers.size() - 1); ICachedPage oldRoot = bufferCache.pin( - BufferedFileHandle.getDiskPageId(fileId, lastNodeFrontier.pageId), false, largePageHelper); + BufferedFileHandle.getDiskPageId(fileId, lastNodeFrontier.pageId), false); oldRoot.acquireReadLatch(); lastNodeFrontier.page = oldRoot; try { diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java index 87449eb..679ea2f 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java @@ -64,14 +64,15 @@ } private boolean positionToNextLeaf(boolean skipCurrent) throws HyracksDataException { - while ((frame.getLevel() != 0 || skipCurrent || frame.getTupleCount() == 0) && (currentPageId <= maxPageId)) { - currentPageId++; + while (frame.getLevel() != 0 || skipCurrent || frame.getTupleCount() == 0) { + if (++currentPageId > maxPageId) { + break; + } page.releaseReadLatch(); bufferCache.unpin(page); - ICachedPage nextPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false, - frame.getLargePageHelper()); + ICachedPage nextPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false); nextPage.acquireReadLatch(); page = nextPage; diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java index d9013d3..5668992 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java @@ -53,8 +53,7 @@ // scan entire file to determine pages in each level int maxPageId = freePageManager.getMaxPage(metaFrame); for (int pageId = 0; pageId <= maxPageId; pageId++) { - ICachedPage page = bufferCache.pin( - BufferedFileHandle.getDiskPageId(fileId, pageId), false); + ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false); page.acquireReadLatch(); try { frame.setPage(page); @@ -89,8 +88,7 @@ int pageId = remainingPageIds[index]; // pin & latch then immediately unlatch & unpin - ICachedPage page = bufferCache.pin(BufferedFileHandle - .getDiskPageId(fileId, pageId), false); + ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false); page.acquireReadLatch(); page.releaseReadLatch(); bufferCache.unpin(page); diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java index 7a9e588..712acb8 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java @@ -52,8 +52,7 @@ int maxPageId = freePageManager.getMaxPage(metaFrame); for (int pageId = 0; pageId <= maxPageId; pageId++) { - ICachedPage page = bufferCache.pin( - BufferedFileHandle.getDiskPageId(fileId, pageId), false, leafFrame.getLargePageHelper()); + ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false); page.acquireReadLatch(); try { metaFrame.setPage(page); diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java index c4fce24..748dbfe 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java @@ -25,8 +25,8 @@ import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.replication.IIOReplicationManager; import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; import org.apache.hyracks.storage.common.buffercache.ICachedPage; +import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue; import org.apache.hyracks.storage.common.file.IFileMapManager; @@ -69,11 +69,6 @@ @Override public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException { return vbc.pin(dpid, newPage); - } - - @Override - public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException { - return vbc.pin(dpid, newPage, helper); } @Override @@ -163,8 +158,9 @@ } @Override - public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException { - return vbc.confiscateLargePage(dpid, multiplier); + public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId) + throws HyracksDataException { + return vbc.confiscateLargePage(dpid, multiplier, extraBlockPageId); } @Override @@ -218,7 +214,8 @@ } @Override - public void resizePage(ICachedPage page, int multiple) { - vbc.resizePage(page, multiple); + public void resizePage(ICachedPage page, int multiplier, IExtraPageBlockHelper extraPageBlockHelper) + throws HyracksDataException { + vbc.resizePage(page, multiplier, extraPageBlockHelper); } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java index e62f824..40c7734 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java @@ -34,8 +34,8 @@ import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; import org.apache.hyracks.storage.common.buffercache.ICacheMemoryAllocator; import org.apache.hyracks.storage.common.buffercache.ICachedPage; +import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; import org.apache.hyracks.storage.common.buffercache.IQueueInfo; import org.apache.hyracks.storage.common.file.BufferedFileHandle; import org.apache.hyracks.storage.common.file.IFileMapManager; @@ -66,7 +66,7 @@ this.numPages = 2 * (numPages / 2) + 1; buckets = new CacheBucket[this.numPages]; - pages = new ArrayList<VirtualPage>(); + pages = new ArrayList<>(); nextFree = 0; largePages = new AtomicInteger(0); open = false; @@ -168,11 +168,6 @@ } @Override - public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException { - return pin(dpid, newPage); - } - - @Override public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException { VirtualPage page = null; int hash = hash(dpid); @@ -213,6 +208,7 @@ synchronized (pages) { if (nextFree >= pages.size()) { page = new VirtualPage(allocator.allocate(pageSize, 1)[0]); + page.multiplier = 1; pages.add(page); } else { page = pages.get(nextFree); @@ -224,9 +220,9 @@ } @Override - public void resizePage(ICachedPage cPage, int multiplier) { + public void resizePage(ICachedPage cPage, int multiplier, IExtraPageBlockHelper extraPageBlockHelper) { ByteBuffer oldBuffer = cPage.getBuffer(); - int origMultiplier = oldBuffer.capacity() / pageSize; + int origMultiplier = cPage.getFrameSizeMultiplier(); if (origMultiplier == multiplier) { // no-op return; @@ -253,6 +249,7 @@ largePages.getAndAdd(multiplier - origMultiplier); } ((VirtualPage)cPage).buffer = newBuffer; + ((VirtualPage)cPage).multiplier = multiplier; } @Override @@ -354,6 +351,7 @@ ByteBuffer buffer; final ReadWriteLock latch; volatile long dpid; + int multiplier; VirtualPage next; public VirtualPage(ByteBuffer buffer) { @@ -406,6 +404,10 @@ throw new UnsupportedOperationException(); } + @Override + public int getFrameSizeMultiplier() { + return multiplier; + } } //These 4 methods aren't applicable here. @@ -447,12 +449,13 @@ } @Override - public ICachedPage confiscatePage(long dpid) { + public ICachedPage confiscatePage(long dpid) throws HyracksDataException { throw new UnsupportedOperationException("Virtual buffer caches don't have FIFO writers"); } @Override - public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException { + public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId) + throws HyracksDataException { throw new UnsupportedOperationException("Virtual buffer caches don't have FIFO writers"); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java index 56827a5..84cb82e 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java @@ -21,11 +21,9 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; -import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager; import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProvider; import org.apache.hyracks.storage.am.common.api.ISplitKey; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; -import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; import org.apache.hyracks.storage.am.common.frames.TreeIndexNSMFrame; @@ -34,6 +32,7 @@ import org.apache.hyracks.storage.am.rtree.api.IRTreePolicy; import org.apache.hyracks.storage.am.rtree.impls.UnorderedSlotManager; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; public abstract class RTreeNSMFrame extends TreeIndexNSMFrame implements IRTreeFrame { protected static final int pageNsnOff = flagOff + 1; // 22 @@ -50,7 +49,7 @@ public RTreeNSMFrame(ITreeIndexTupleWriter tupleWriter, IPrimitiveValueProvider[] keyValueProviders, RTreePolicyType rtreePolicyType, boolean isPointMBR) { - super(tupleWriter, new UnorderedSlotManager(), null); + super(tupleWriter, new UnorderedSlotManager()); this.mbrTuples = new ITreeIndexTupleReference[keyValueProviders.length]; for (int i = 0; i < keyValueProviders.length; i++) { this.mbrTuples[i] = tupleWriter.createTupleReference(); @@ -116,7 +115,7 @@ @Override public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey, - IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache) + IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache) throws HyracksDataException { rtreePolicy.split(this, buf, rightFrame, slotManager, frameTuple, tuple, splitKey); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java index 02177f4..8798a70 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java @@ -24,7 +24,6 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory; import org.apache.hyracks.storage.am.rtree.api.IRTreeInteriorFrame; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; public class RTreeNSMInteriorFrameFactory implements ITreeIndexFrameFactory { @@ -61,8 +60,4 @@ return tupleWriterFactory; } - @Override - public ILargePageHelper getLargePageHelper() { - return null; - } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java index 6373c7e..8573119 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java @@ -24,7 +24,6 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory; import org.apache.hyracks.storage.am.rtree.api.IRTreeLeafFrame; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; public class RTreeNSMLeafFrameFactory implements ITreeIndexFrameFactory { @@ -61,8 +60,4 @@ return tupleWriterFactory; } - @Override - public ILargePageHelper getLargePageHelper() { - return null; - } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java index 84180aa..99edbc4 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java @@ -155,8 +155,8 @@ private RTreeOpContext createOpContext(IModificationOperationCallback modificationCallback) { return new RTreeOpContext((IRTreeLeafFrame) leafFrameFactory.createFrame(), - (IRTreeInteriorFrame) interiorFrameFactory.createFrame(), - freePageManager.getMetaDataFrameFactory().createFrame(), cmpFactories, modificationCallback); + (IRTreeInteriorFrame) interiorFrameFactory.createFrame(), freePageManager, cmpFactories, + modificationCallback); } private void insert(ITupleReference tuple, IIndexOperationContext ictx) @@ -412,7 +412,7 @@ rightFrame.setPage(rightNode); rightFrame.initBuffer((byte) ctx.interiorFrame.getLevel()); rightFrame.setRightPage(ctx.interiorFrame.getRightPage()); - ctx.interiorFrame.split(rightFrame, tuple, ctx.splitKey, freePageManager, ctx.metaFrame, + ctx.interiorFrame.split(rightFrame, tuple, ctx.splitKey, ctx, bufferCache); ctx.interiorFrame.setRightPage(rightPageId); } else { @@ -421,7 +421,7 @@ rightFrame.initBuffer((byte) 0); rightFrame.setRightPage(ctx.interiorFrame.getRightPage()); ctx.modificationCallback.found(null, tuple); - ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey, freePageManager, ctx.metaFrame, + ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey, ctx, bufferCache); ctx.leafFrame.setRightPage(rightPageId); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java index 107dc30..a04562f 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java @@ -22,8 +22,10 @@ import java.util.ArrayList; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.common.api.IIndexOperationContext; +import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager; import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback; import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor; import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame; @@ -32,8 +34,9 @@ import org.apache.hyracks.storage.am.rtree.api.IRTreeInteriorFrame; import org.apache.hyracks.storage.am.rtree.api.IRTreeLeafFrame; import org.apache.hyracks.storage.common.buffercache.ICachedPage; +import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; -public class RTreeOpContext implements IIndexOperationContext { +public class RTreeOpContext implements IIndexOperationContext, IExtraPageBlockHelper { private static final int INITIAL_TRAVERSE_LIST_SIZE = 100; private static final int INITIAL_HEIGHT = 8; public final MultiComparator cmp; @@ -42,7 +45,8 @@ public IndexOperation op; public ITreeIndexCursor cursor; public RTreeCursorInitialState cursorInitialState; - public ITreeIndexMetaDataFrame metaFrame; + public final IMetaDataPageManager freePageManager; + public final ITreeIndexMetaDataFrame metaFrame; public RTreeSplitKey splitKey; public ITupleReference tuple; // Used to record the pageIds and pageLsns of the visited pages. @@ -56,8 +60,8 @@ public IModificationOperationCallback modificationCallback; public RTreeOpContext(IRTreeLeafFrame leafFrame, IRTreeInteriorFrame interiorFrame, - ITreeIndexMetaDataFrame metaFrame, IBinaryComparatorFactory[] cmpFactories, - IModificationOperationCallback modificationCallback) { + IMetaDataPageManager freePageManager, IBinaryComparatorFactory[] cmpFactories, + IModificationOperationCallback modificationCallback) { if (cmpFactories[0] != null) { this.cmp = MultiComparator.create(cmpFactories); @@ -67,7 +71,8 @@ this.interiorFrame = interiorFrame; this.leafFrame = leafFrame; - this.metaFrame = metaFrame; + this.freePageManager = freePageManager; + this.metaFrame = freePageManager.getMetaDataFrameFactory().createFrame(); this.modificationCallback = modificationCallback; pathList = new PathList(INITIAL_HEIGHT, INITIAL_HEIGHT); NSNUpdates = new ArrayList<ICachedPage>(); @@ -121,4 +126,14 @@ public void setModificationCallback(IModificationOperationCallback modificationCallback) { this.modificationCallback = modificationCallback; } + + @Override + public int getFreeBlock(int size) throws HyracksDataException { + return freePageManager.getFreePageBlock(metaFrame, size); + } + + @Override + public void returnFreePageBlock(int blockPageId, int size) throws HyracksDataException { + freePageManager.addFreePageBlock(metaFrame, blockPageId, size); + } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java index 201526b..ceb8ca2 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java @@ -28,8 +28,10 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; @@ -49,6 +51,7 @@ import org.apache.hyracks.storage.common.file.IFileMapManager; public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { + private static final Logger LOGGER = Logger.getLogger(BufferCache.class.getName()); private static final int MAP_FACTOR = 3; @@ -68,6 +71,8 @@ private final Map<Integer, BufferedFileHandle> fileInfoMap; private final Set<Integer> virtualFiles; private final AsyncFIFOPageQueueManager fifoWriter; + private final Queue<BufferCacheHeaderHelper> headerPageCache = new ConcurrentLinkedQueue<>(); + //DEBUG private ArrayList<CachedPage> confiscatedPages; private Lock confiscateLock; @@ -124,6 +129,10 @@ return pageSize; } + public int getPageSizeWithHeader() { + return pageSize + RESERVED_HEADER_BYTES; + } + @Override public int getNumPages() { return pageReplacementStrategy.getMaxAllowedNumPages(); @@ -176,11 +185,6 @@ @Override public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException { - return pin(dpid, newPage, null); - } - - @Override - public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException { // Calling the pinSanityCheck should be used only for debugging, since // the synchronized block over the fileInfoMap is a hot spot. if (DEBUG) { @@ -193,9 +197,7 @@ try { for (CachedPage c : confiscatedPages) { if (c.dpid == dpid && c.confiscated.get()) { - while(confiscatedPages.contains(c)){ - throw new IllegalStateException(); - } + throw new IllegalStateException(); } } } finally{ @@ -206,7 +208,7 @@ // disk. synchronized (cPage) { if (!cPage.valid) { - read(cPage, helper); + read(cPage); cPage.valid = true; } } @@ -215,9 +217,8 @@ } pageReplacementStrategy.notifyCachePageAccess(cPage); if(DEBUG){ - pinnedPageOwner.put((CachedPage) cPage, Thread.currentThread().getStackTrace()); + pinnedPageOwner.put(cPage, Thread.currentThread().getStackTrace()); } - cPage.setLargePageHelper(helper); return cPage; } @@ -565,26 +566,42 @@ return false; } - private void read(CachedPage cPage, ILargePageHelper helper) throws HyracksDataException { + private void read(CachedPage cPage) throws HyracksDataException { BufferedFileHandle fInfo = getFileInfo(cPage); cPage.buffer.clear(); - ioManager.syncRead(fInfo.getFileHandle(), (long) BufferedFileHandle.getPageId(cPage.dpid) * pageSize, - cPage.buffer); - if (helper != null) { - int totalPages = helper.getSupplementalBlockNumPages(cPage) + 1; + BufferCacheHeaderHelper header = checkoutHeaderHelper(); + try { + long bytesRead = ioManager.syncRead(fInfo.getFileHandle(), + getOffsetForPage(BufferedFileHandle.getPageId(cPage.dpid)), header.prepareRead()); + + if (bytesRead != getPageSizeWithHeader()) { + if (bytesRead == -1) { + // disk order scan code seems to rely on this behavior, so silently return + return; + } + throw new HyracksDataException("Failed to read a complete page: " + bytesRead); + } + int totalPages = header.processRead(cPage); + if (totalPages > 1) { - resizePage(cPage, totalPages); + pageReplacementStrategy.fixupCapacityOnLargeRead(cPage); cPage.buffer.position(pageSize); cPage.buffer.limit(totalPages * pageSize); - ioManager.syncRead(fInfo.getFileHandle(), (long) helper.getSupplementalBlockPageId(cPage) * pageSize, - cPage.buffer); + ioManager.syncRead(fInfo.getFileHandle(), getOffsetForPage(cPage.getExtraBlockPageId()), cPage.buffer); } + } finally { + returnHeaderHelper(header); } } + private long getOffsetForPage(long pageId) { + return pageId * getPageSizeWithHeader(); + } + @Override - public void resizePage(ICachedPage cPage, int totalPages) { - pageReplacementStrategy.resizePage((ICachedPageInternal) cPage, totalPages); + public void resizePage(ICachedPage cPage, int totalPages, IExtraPageBlockHelper extraPageBlockHelper) + throws HyracksDataException { + pageReplacementStrategy.resizePage((ICachedPageInternal) cPage, totalPages, extraPageBlockHelper); } BufferedFileHandle getFileInfo(CachedPage cPage) throws HyracksDataException { @@ -600,6 +617,17 @@ return fInfo; } } + private BufferCacheHeaderHelper checkoutHeaderHelper() { + BufferCacheHeaderHelper helper = headerPageCache.poll(); + if (helper == null) { + helper = new BufferCacheHeaderHelper(pageSize); + } + return helper; + } + + private void returnHeaderHelper(BufferCacheHeaderHelper buffer) { + headerPageCache.offer(buffer); + } void write(CachedPage cPage) throws HyracksDataException { BufferedFileHandle fInfo = getFileInfo(cPage); @@ -607,21 +635,29 @@ synchronized (fInfo) { if (!fInfo.fileHasBeenDeleted()) { ByteBuffer buf = cPage.buffer.duplicate(); - buf.position(0); - buf.limit(pageSize); - ioManager.syncWrite(fInfo.getFileHandle(), (long) BufferedFileHandle.getPageId(cPage.dpid) * pageSize, - buf); - if (cPage.largePageHelper != null) { - int totalPages = cPage.largePageHelper.getSupplementalBlockNumPages(cPage) + 1; - if (totalPages > 1) { - buf.limit(totalPages * pageSize); - long offset = (long) cPage.largePageHelper.getSupplementalBlockPageId(cPage) * pageSize; - ioManager.syncWrite(fInfo.getFileHandle(), offset, buf); + final int totalPages = cPage.getFrameSizeMultiplier(); + final int extraBlockPageId = cPage.getExtraBlockPageId(); + final boolean contiguousLargePages = (BufferedFileHandle.getPageId(cPage.dpid) + 1) == extraBlockPageId; + BufferCacheHeaderHelper header = checkoutHeaderHelper(); + try { + buf.limit(contiguousLargePages ? pageSize * totalPages : pageSize); + buf.position(0); + long bytesWritten = ioManager.syncWrite(fInfo.getFileHandle(), + getOffsetForPage(BufferedFileHandle.getPageId(cPage.dpid)), + header.prepareWrite(cPage, buf)); + + if (bytesWritten != + (contiguousLargePages ? pageSize * (totalPages - 1) : 0) + getPageSizeWithHeader()) { + throw new HyracksDataException("Failed to write completely: " + bytesWritten); } - assert buf.capacity() == (pageSize * totalPages); - } else { - assert buf.capacity() == pageSize; + } finally { + returnHeaderHelper(header); } + if (totalPages > 1 && !contiguousLargePages) { + buf.limit(totalPages * pageSize); + ioManager.syncWrite(fInfo.getFileHandle(), getOffsetForPage(extraBlockPageId), buf); + } + assert buf.capacity() == (pageSize * totalPages); } } } @@ -1134,9 +1170,9 @@ throw new HyracksDataException("No such file mapped for fileId:" + fileId); } if(DEBUG) { - assert ioManager.getSize(fInfo.getFileHandle()) % getPageSize() == 0; + assert ioManager.getSize(fInfo.getFileHandle()) % getPageSizeWithHeader() == 0; } - return (int) (ioManager.getSize(fInfo.getFileHandle()) / getPageSize()); + return (int) (ioManager.getSize(fInfo.getFileHandle()) / getPageSizeWithHeader()); } } @@ -1151,8 +1187,11 @@ } @Override - public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException { - return confiscatePage(dpid, () -> pageReplacementStrategy.findVictim(multiplier)); + public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId) + throws HyracksDataException { + ICachedPage cachedPage = confiscatePage(dpid, () -> pageReplacementStrategy.findVictim(multiplier)); + ((ICachedPageInternal)cachedPage).setExtraBlockPageId(extraBlockPageId); + return cachedPage; } private ICachedPage confiscatePage(long dpid, Supplier<ICachedPageInternal> victimSupplier) @@ -1236,7 +1275,6 @@ // if we found a page after all that, go ahead and finish if (returnPage != null) { ((CachedPage) returnPage).confiscated.set(true); - ((CachedPage) returnPage).setLargePageHelper(null); if (DEBUG) { confiscateLock.lock(); try{ @@ -1336,7 +1374,6 @@ } } pageReplacementStrategy.adviseWontNeed(cPage); - cPage.largePageHelper = null; } @Override @@ -1390,4 +1427,41 @@ } } + static class BufferCacheHeaderHelper { + private static final int FRAME_MULTIPLIER_OFF = 0; + private static final int EXTRA_BLOCK_PAGE_ID_OFF = FRAME_MULTIPLIER_OFF + 4; // 4 + + private final ByteBuffer buf; + private final ByteBuffer [] array; + + private BufferCacheHeaderHelper(int pageSize) { + buf = ByteBuffer.allocate(RESERVED_HEADER_BYTES + pageSize); + array = new ByteBuffer[] { buf, null }; + } + + private ByteBuffer[] prepareWrite(CachedPage cPage, ByteBuffer pageBuffer) { + buf.position(0); + buf.limit(RESERVED_HEADER_BYTES); + buf.putInt(FRAME_MULTIPLIER_OFF, cPage.getFrameSizeMultiplier()); + buf.putInt(EXTRA_BLOCK_PAGE_ID_OFF, cPage.getExtraBlockPageId()); + array[1] = pageBuffer; + return array; + } + + private ByteBuffer prepareRead() { + buf.position(0); + buf.limit(buf.capacity()); + return buf; + } + + private int processRead(CachedPage cPage) { + buf.position(RESERVED_HEADER_BYTES); + cPage.buffer.position(0); + cPage.buffer.put(buf); + int multiplier = buf.getInt(FRAME_MULTIPLIER_OFF); + cPage.setFrameSizeMultiplier(multiplier); + cPage.setExtraBlockPageId(buf.getInt(EXTRA_BLOCK_PAGE_ID_OFF)); + return multiplier; + } + } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java index ddab742..4541d52 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java @@ -39,7 +39,11 @@ volatile boolean valid; final AtomicBoolean confiscated; private IQueueInfo queueInfo; - ILargePageHelper largePageHelper; + private int multiplier; + private int extraBlockPageId; + // DEBUG + private static final boolean DEBUG = false; + private final StackTraceElement[] ctorStack; //Constructor for making dummy entry for FIFO queue public CachedPage(){ @@ -52,6 +56,7 @@ queueInfo = null; replacementStrategyObject = null; latch =null; + ctorStack = DEBUG ? new Throwable().getStackTrace() : null; } public CachedPage(int cpid, ByteBuffer buffer, IPageReplacementStrategy pageReplacementStrategy) { @@ -66,6 +71,7 @@ valid = false; confiscated = new AtomicBoolean(false); queueInfo = null; + ctorStack = DEBUG ? new Throwable().getStackTrace() : null; } public void reset(long dpid) { @@ -75,7 +81,6 @@ confiscated.set(false); pageReplacementStrategy.notifyCachePageReset(this); queueInfo = null; - largePageHelper = null; } public void invalidate() { @@ -154,6 +159,26 @@ return dpid; } + @Override + public int getFrameSizeMultiplier() { + return multiplier; + } + + @Override + public void setFrameSizeMultiplier(int multiplier) { + this.multiplier = multiplier; + } + + @Override + public void setExtraBlockPageId(int extraBlockPageId) { + this.extraBlockPageId = extraBlockPageId; + } + + @Override + public int getExtraBlockPageId() { + return extraBlockPageId; + } + CachedPage getNext() { return next; } @@ -161,13 +186,4 @@ void setNext(CachedPage next) { this.next = next; } - - public ILargePageHelper getLargePageHelper() { - return largePageHelper; - } - - public void setLargePageHelper(ILargePageHelper largePageHelper) { - this.largePageHelper = largePageHelper; - } - } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java index ea27757..e397d66 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java @@ -25,6 +25,8 @@ import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.hyracks.api.exceptions.HyracksDataException; + public class ClockPageReplacementStrategy implements IPageReplacementStrategy { private static final Logger LOGGER = Logger.getLogger(ClockPageReplacementStrategy.class.getName()); private static final int MAX_UNSUCCESSFUL_CYCLE_COUNT = 3; @@ -80,13 +82,13 @@ @Override public ICachedPageInternal findVictim(int multiplier) { - while (numPages.get() + (multiplier - 1) >= maxAllowedNumPages) { + while (numPages.get() + multiplier > maxAllowedNumPages) { // TODO: is dropping pages on the floor enough to adhere to memory budget? ICachedPageInternal victim = findVictimByEviction(); if (victim == null) { return null; } - int multiple = victim.getBuffer().capacity() / pageSize; + int multiple = victim.getFrameSizeMultiplier(); if (multiple == multiplier) { return victim; } else if (bufferCache.removePage(victim)) { @@ -153,50 +155,79 @@ cpId = cpIdCounter.getAndIncrement(); } CachedPage cPage = new CachedPage(cpId, allocator.allocate(pageSize * multiplier, 1)[0], this); + cPage.setFrameSizeMultiplier(multiplier); bufferCache.addPage(cPage); numPages.getAndAdd(multiplier); return cPage; } @Override - public void resizePage(ICachedPageInternal cPage, int multiplier) { - ByteBuffer oldBuffer = cPage.getBuffer(); - int origMultiplier = oldBuffer.capacity() / pageSize; + public void resizePage(ICachedPageInternal cPage, int multiplier, IExtraPageBlockHelper extraPageBlockHelper) + throws HyracksDataException { + int origMultiplier = cPage.getFrameSizeMultiplier(); if (origMultiplier == multiplier) { // no-op return; } final int newSize = pageSize * multiplier; + ByteBuffer oldBuffer = ((CachedPage)cPage).buffer; oldBuffer.position(0); + final int delta = multiplier - origMultiplier; if (multiplier < origMultiplier) { oldBuffer.limit(newSize); + final int gap = -delta; + // we return the unused portion of our block to the page manager + extraPageBlockHelper.returnFreePageBlock(cPage.getExtraBlockPageId() + gap, gap); } else { - while (numPages.get() + (multiplier - origMultiplier) >= maxAllowedNumPages) { - ICachedPageInternal victim = findVictimByEviction(); - if (victim != null) { - int multiple = victim.getBuffer().capacity() / pageSize; - if (bufferCache.removePage(victim)) { - // TODO(mblow): is dropping pages on the floor enough to adhere to memory budget? - cpIdFreeList.add(victim.getCachedPageId()); - numPages.getAndAdd(-multiple); - } - } else { - // TODO(mblow): what should we do with we need to resize and don't have the budget? - // we don't have the budget to resize- proceed anyway, but log - if (LOGGER.isLoggable(Level.WARNING)) { - LOGGER.warning("Exceeding buffer cache budget of " + maxAllowedNumPages + " by " - + (numPages.get() + (multiplier - origMultiplier) - maxAllowedNumPages) - + " pages in order to satisfy large page read"); - } - break; - - } + ensureBudgetForLargePages(delta); + if (origMultiplier != 1) { + // return the old block to the page manager + extraPageBlockHelper.returnFreePageBlock(cPage.getExtraBlockPageId(), origMultiplier); } + cPage.setExtraBlockPageId(extraPageBlockHelper.getFreeBlock(multiplier)); } + cPage.setFrameSizeMultiplier(multiplier); ByteBuffer newBuffer = allocator.allocate(newSize, 1)[0]; newBuffer.put(oldBuffer); - numPages.getAndAdd(multiplier - origMultiplier); + numPages.getAndAdd(delta); ((CachedPage) cPage).buffer = newBuffer; + } + + @Override + public void fixupCapacityOnLargeRead(ICachedPageInternal cPage) + throws HyracksDataException { + ByteBuffer oldBuffer = ((CachedPage) cPage).buffer; + final int multiplier = cPage.getFrameSizeMultiplier(); + final int newSize = pageSize * multiplier; + final int delta = multiplier - 1; + oldBuffer.position(0); + ensureBudgetForLargePages(delta); + ByteBuffer newBuffer = allocator.allocate(newSize, 1)[0]; + newBuffer.put(oldBuffer); + numPages.getAndAdd(delta); + ((CachedPage) cPage).buffer = newBuffer; + } + + private void ensureBudgetForLargePages(int delta) { + while (numPages.get() + delta >= maxAllowedNumPages) { + ICachedPageInternal victim = findVictimByEviction(); + if (victim != null) { + final int victimMultiplier = victim.getFrameSizeMultiplier(); + if (bufferCache.removePage(victim)) { + cpIdFreeList.add(victim.getCachedPageId()); + numPages.getAndAdd(-victimMultiplier); + } + } else { + // we don't have the budget to resize- proceed anyway, but log + if (LOGGER.isLoggable(Level.WARNING)) { + LOGGER.warning("Exceeding buffer cache budget of " + maxAllowedNumPages + " by " + + (numPages.get() + delta - maxAllowedNumPages) + + " pages in order to satisfy large page read"); + } + break; + + } + } } //derived from RoundRobinAllocationPolicy in Apache directmemory @@ -232,4 +263,5 @@ //make the page appear as if it wasn't accessed even if it was getPerPageObject(cPage).set(false); } + } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java index c78012b..4a41ec0 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java @@ -88,13 +88,6 @@ } @Override - public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException { - ICachedPage page = bufferCache.pin(dpid, newPage, helper); - pinCount.addAndGet(1); - return page; - } - - @Override public void unpin(ICachedPage page) throws HyracksDataException { bufferCache.unpin(page); unpinCount.addAndGet(1); @@ -204,8 +197,9 @@ } @Override - public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException { - return bufferCache.confiscateLargePage(dpid, multiplier); + public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId) + throws HyracksDataException { + return bufferCache.confiscateLargePage(dpid, multiplier, extraBlockPageId); } @Override @@ -262,7 +256,8 @@ } @Override - public void resizePage(ICachedPage page, int multiple) { - bufferCache.resizePage(page, multiple); + public void resizePage(ICachedPage page, int multiplier, IExtraPageBlockHelper extraPageBlockHelper) + throws HyracksDataException { + bufferCache.resizePage(page, multiplier, extraPageBlockHelper); } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java index feac69e..ca87673 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java @@ -24,8 +24,9 @@ public interface IBufferCache { - public static final long INVALID_DPID = -1l; - public static final int INVALID_PAGEID = -1; + long INVALID_DPID = -1l; + int INVALID_PAGEID = -1; + int RESERVED_HEADER_BYTES = 8; public void createFile(FileReference fileRef) throws HyracksDataException; @@ -43,8 +44,6 @@ public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException; - public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException; - public void unpin(ICachedPage page) throws HyracksDataException; public void flushDirtyPage(ICachedPage page) throws HyracksDataException; @@ -53,7 +52,7 @@ public ICachedPage confiscatePage(long dpid) throws HyracksDataException; - public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException; + public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId) throws HyracksDataException; public void returnPage(ICachedPage page); @@ -61,7 +60,7 @@ public void force(int fileId, boolean metadata) throws HyracksDataException; - public int getPageSize(); + int getPageSize(); public int getNumPages(); @@ -85,5 +84,6 @@ void purgeHandle(int fileId) throws HyracksDataException; - void resizePage(ICachedPage page, int multiple); + void resizePage(ICachedPage page, int multiplier, IExtraPageBlockHelper extraPageBlockHelper) + throws HyracksDataException; } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java index 12dde49..8107234 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java @@ -21,19 +21,22 @@ import java.nio.ByteBuffer; public interface ICachedPage { - public ByteBuffer getBuffer(); - public void acquireReadLatch(); + ByteBuffer getBuffer(); - public void releaseReadLatch(); + void acquireReadLatch(); - public void acquireWriteLatch(); + void releaseReadLatch(); - public void releaseWriteLatch(boolean markDirty); + void acquireWriteLatch(); - public boolean confiscated(); + void releaseWriteLatch(boolean markDirty); - public IQueueInfo getQueueInfo(); + boolean confiscated(); - public void setQueueInfo(IQueueInfo queueInfo); + IQueueInfo getQueueInfo(); + + void setQueueInfo(IQueueInfo queueInfo); + + int getFrameSizeMultiplier(); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java index 6a6c2f5..c500286 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java @@ -27,4 +27,9 @@ public boolean isGoodVictim(); + void setFrameSizeMultiplier(int multiplier); + + int getExtraBlockPageId(); + + void setExtraBlockPageId(int extraBlockPageId); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IExtraPageBlockHelper.java similarity index 77% copy from hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java copy to hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IExtraPageBlockHelper.java index 22bec97..ad7f2f6 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IExtraPageBlockHelper.java @@ -18,8 +18,9 @@ */ package org.apache.hyracks.storage.common.buffercache; -public interface ILargePageHelper { +import org.apache.hyracks.api.exceptions.HyracksDataException; - int getSupplementalBlockNumPages(CachedPage cPage); - int getSupplementalBlockPageId(CachedPage cPage); +public interface IExtraPageBlockHelper { + int getFreeBlock(int size) throws HyracksDataException; + void returnFreePageBlock(int blockPageId, int size) throws HyracksDataException; } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java index 1a59a65..d3bcce5 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java @@ -18,6 +18,8 @@ */ package org.apache.hyracks.storage.common.buffercache; +import org.apache.hyracks.api.exceptions.HyracksDataException; + public interface IPageReplacementStrategy { public Object createPerPageStrategyObject(int cpid); @@ -37,9 +39,13 @@ public int getNumPages(); + void fixupCapacityOnLargeRead(ICachedPageInternal cPage) + throws HyracksDataException; + public int getPageSize(); public int getMaxAllowedNumPages(); - public void resizePage(ICachedPageInternal page, int multiplier); + void resizePage(ICachedPageInternal page, int multiplier, IExtraPageBlockHelper extraPageBlockHelper) + throws HyracksDataException; } diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java index 840acd5..4420fb0 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java @@ -23,7 +23,6 @@ import java.util.Random; import java.util.logging.Level; -import org.apache.hyracks.storage.am.btree.frames.BTreeLargeFrameHelper; import org.junit.Assert; import org.junit.Test; @@ -136,8 +135,7 @@ int btreeFileId = fileMapProvider.lookupFileId(harness.getFileReference()); bufferCache.openFile(btreeFileId); IHyracksTaskContext ctx = harness.getHyracksTaskContext(); - ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(btreeFileId, 0), false, - BTreeLargeFrameHelper.INSTANCE); + ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(btreeFileId, 0), true); try { ITreeIndexTupleWriter tupleWriter = new TypeAwareTupleWriter(typeTraits); diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java index cce2f82..01863ca 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java @@ -24,7 +24,6 @@ import java.util.Random; import java.util.logging.Level; -import org.apache.hyracks.storage.am.btree.frames.BTreeLargeFrameHelper; import org.junit.Test; import org.apache.hyracks.api.exceptions.HyracksDataException; @@ -90,8 +89,7 @@ } try { - ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, - BTreeLargeFrameHelper.INSTANCE); + ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false); LatchType latch = null; switch (fta) { diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java index 2330e4d..e85ff9e 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java @@ -18,15 +18,9 @@ */ package org.apache.hyracks.storage.common; -import static org.junit.Assert.fail; - import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.io.FileReference; @@ -34,12 +28,18 @@ import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.api.io.IIOManager.FileReadWriteMode; import org.apache.hyracks.api.io.IIOManager.FileSyncMode; +import org.apache.hyracks.storage.common.buffercache.BufferCache; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.file.BufferedFileHandle; import org.apache.hyracks.storage.common.file.IFileMapProvider; import org.apache.hyracks.test.support.TestStorageManagerComponentHolder; import org.apache.hyracks.test.support.TestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.fail; public class BufferCacheRegressionTest { protected static final String tmpDir = System.getProperty("java.io.tmpdir"); @@ -126,9 +126,9 @@ FileReference testFileRef = new FileReference(new File(fileName)); IFileHandle testFileHandle = ioManager.open(testFileRef, FileReadWriteMode.READ_ONLY, FileSyncMode.METADATA_SYNC_DATA_SYNC); - ByteBuffer testBuffer = ByteBuffer.allocate(PAGE_SIZE); + ByteBuffer testBuffer = ByteBuffer.allocate(PAGE_SIZE + BufferCache.RESERVED_HEADER_BYTES); ioManager.syncRead(testFileHandle, 0, testBuffer); - for (int i = 0; i < testBuffer.capacity(); i++) { + for (int i = BufferCache.RESERVED_HEADER_BYTES; i < testBuffer.capacity(); i++) { if (deleteFile) { // We deleted the file. We expect to see a clean buffer. if (testBuffer.get(i) == Byte.MAX_VALUE) { -- To view, visit https://asterix-gerrit.ics.uci.edu/920 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd Gerrit-PatchSet: 5 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Michael Blow <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
