Repository: asterixdb Updated Branches: refs/heads/master 323403bab -> b8ea58cd8
[NO ISSUE] Misc Fixes for Point MBR RTree - user-model changes: no - storage format changes: no - interface changes: no Details: 1. Fix the MBR calculation of Point MBR RTrees when finalizing the bulkload process. 2. Fix the nullFlagBytes of Point MBR Rtrees. 3. Add unit test cases to cover Point MBR RTrees. Change-Id: Ice24112152a2d93c7d2316b7506e6d6e81f2df44 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2911 Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: abdullah alamoudi <bamou...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/b8ea58cd Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/b8ea58cd Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/b8ea58cd Branch: refs/heads/master Commit: b8ea58cd85077c0d28d746cda49be7dc946ff6e1 Parents: 323403b Author: luochen01 <cl...@uci.edu> Authored: Wed Aug 15 11:32:05 2018 -0700 Committer: Luo Chen <cl...@uci.edu> Committed: Wed Aug 15 14:20:22 2018 -0700 ---------------------------------------------------------------------- .../LSMRTreeTupleReferenceForPointMBR.java | 4 +- .../hyracks/storage/am/rtree/impls/RTree.java | 1 + .../am/rtree/AbstractRTreeBulkLoadTest.java | 10 ++- .../storage/am/rtree/RTreeTestUtils.java | 75 ++++++++++++++++---- .../lsm/rtree/LSMRTreePointMBRBulkLoadTest.java | 70 ++++++++++++++++++ .../am/lsm/rtree/util/LSMRTreeTestContext.java | 13 +++- 6 files changed, 156 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8ea58cd/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleReferenceForPointMBR.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleReferenceForPointMBR.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleReferenceForPointMBR.java index 1432aba..e28d5a2 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleReferenceForPointMBR.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleReferenceForPointMBR.java @@ -128,7 +128,9 @@ public class LSMRTreeTupleReferenceForPointMBR extends RTreeTypeAwareTupleRefere @Override protected int getNullFlagsBytes() { - return BitOperationUtils.getFlagBytes(inputTotalFieldCount + (antimatterAware ? 1 : 0)); + // stored key field count + value field count + return BitOperationUtils.getFlagBytes( + storedKeyFieldCount + inputTotalFieldCount - inputKeyFieldCount + (antimatterAware ? 1 : 0)); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8ea58cd/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java index f12f423..635fe7a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java @@ -1006,6 +1006,7 @@ public class RTree extends AbstractTreeIndex { //set next guide MBR //if propagateBulk didnt have to do anything this may be un-necessary if (nodeFrontiers.size() > 1 && nodeFrontiers.indexOf(n) < nodeFrontiers.size() - 1) { + lowerFrame = nodeFrontiers.indexOf(n) != 0 ? prevInteriorFrame : leafFrame; lowerFrame.setPage(n.page); ((RTreeNSMFrame) lowerFrame).adjustMBR(); interiorFrameTupleWriter.writeTupleFields(((RTreeNSMFrame) lowerFrame).getMBRTuples(), 0, mbr, 0); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8ea58cd/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/AbstractRTreeBulkLoadTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/AbstractRTreeBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/AbstractRTreeBulkLoadTest.java index 67b24a2..2d6f97c 100644 --- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/AbstractRTreeBulkLoadTest.java +++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/AbstractRTreeBulkLoadTest.java @@ -30,10 +30,16 @@ import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType; public abstract class AbstractRTreeBulkLoadTest extends AbstractRTreeTestDriver { private final RTreeTestUtils rTreeTestUtils; + private final boolean isPoint; public AbstractRTreeBulkLoadTest(boolean testRstarPolicy) { + this(testRstarPolicy, false); + } + + public AbstractRTreeBulkLoadTest(boolean testRstarPolicy, boolean isPoint) { super(testRstarPolicy); this.rTreeTestUtils = new RTreeTestUtils(); + this.isPoint = isPoint; } @Override @@ -47,9 +53,9 @@ public abstract class AbstractRTreeBulkLoadTest extends AbstractRTreeTestDriver // We assume all fieldSerdes are of the same type. Check the first // one to determine which field types to generate. if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) { - rTreeTestUtils.bulkLoadIntTuples(ctx, numTuplesToInsert, getRandom()); + rTreeTestUtils.bulkLoadIntTuples(ctx, numTuplesToInsert, getRandom(), isPoint); } else if (fieldSerdes[0] instanceof DoubleSerializerDeserializer) { - rTreeTestUtils.bulkLoadDoubleTuples(ctx, numTuplesToInsert, getRandom()); + rTreeTestUtils.bulkLoadDoubleTuples(ctx, numTuplesToInsert, getRandom(), isPoint); } rTreeTestUtils.checkScan(ctx); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8ea58cd/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/RTreeTestUtils.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/RTreeTestUtils.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/RTreeTestUtils.java index 165bf43..92f5055 100644 --- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/RTreeTestUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/RTreeTestUtils.java @@ -93,8 +93,13 @@ public class RTreeTestUtils extends TreeIndexTestUtils { } } + public void insertDoubleTuples(IIndexTestContext ctx, int numTuples, Random rnd) throws HyracksDataException { + insertDoubleTuples(ctx, numTuples, rnd, false); + } + @SuppressWarnings("unchecked") - public void insertDoubleTuples(IIndexTestContext ctx, int numTuples, Random rnd) throws Exception { + public void insertDoubleTuples(IIndexTestContext ctx, int numTuples, Random rnd, boolean isPoint) + throws HyracksDataException { int fieldCount = ctx.getFieldCount(); int numKeyFields = ctx.getKeyFieldCount(); double[] fieldValues = new double[ctx.getFieldCount()]; @@ -104,7 +109,7 @@ public class RTreeTestUtils extends TreeIndexTestUtils { double maxValue = Math.ceil(Math.pow(numTuples, 1.0 / numKeyFields)); for (int i = 0; i < numTuples; i++) { // Set keys. - setDoubleKeyFields(fieldValues, numKeyFields, maxValue, rnd); + setDoubleKeyFields(fieldValues, numKeyFields, maxValue, rnd, isPoint); // Set values. setDoublePayloadFields(fieldValues, numKeyFields, fieldCount); TupleUtils.createDoubleTuple(ctx.getTupleBuilder(), ctx.getTuple(), fieldValues); @@ -126,15 +131,20 @@ public class RTreeTestUtils extends TreeIndexTestUtils { } } - private void setDoubleKeyFields(double[] fieldValues, int numKeyFields, double maxValue, Random rnd) { + private void setDoubleKeyFields(double[] fieldValues, int numKeyFields, double maxValue, Random rnd, + boolean isPoint) { int maxFieldPos = numKeyFields / 2; for (int j = 0; j < maxFieldPos; j++) { int k = maxFieldPos + j; double firstValue = rnd.nextDouble() % maxValue; double secondValue; - do { - secondValue = rnd.nextDouble() % maxValue; - } while (secondValue < firstValue); + if (isPoint) { + secondValue = firstValue; + } else { + do { + secondValue = rnd.nextDouble() % maxValue; + } while (secondValue < firstValue); + } fieldValues[j] = firstValue; fieldValues[k] = secondValue; } @@ -155,8 +165,13 @@ public class RTreeTestUtils extends TreeIndexTestUtils { return checkTuple; } - @SuppressWarnings("unchecked") public void bulkLoadDoubleTuples(IIndexTestContext ctx, int numTuples, Random rnd) throws Exception { + bulkLoadDoubleTuples(ctx, numTuples, rnd, false); + } + + @SuppressWarnings("unchecked") + public void bulkLoadDoubleTuples(IIndexTestContext ctx, int numTuples, Random rnd, boolean isPoint) + throws HyracksDataException { int fieldCount = ctx.getFieldCount(); int numKeyFields = ctx.getKeyFieldCount(); double[] fieldValues = new double[ctx.getFieldCount()]; @@ -164,7 +179,7 @@ public class RTreeTestUtils extends TreeIndexTestUtils { Collection<CheckTuple> tmpCheckTuples = createCheckTuplesCollection(); for (int i = 0; i < numTuples; i++) { // Set keys. - setDoubleKeyFields(fieldValues, numKeyFields, maxValue, rnd); + setDoubleKeyFields(fieldValues, numKeyFields, maxValue, rnd, isPoint); // Set values. setDoublePayloadFields(fieldValues, numKeyFields, fieldCount); @@ -222,22 +237,56 @@ public class RTreeTestUtils extends TreeIndexTestUtils { return checkTuple; } - @Override - protected void setIntKeyFields(int[] fieldValues, int numKeyFields, int maxValue, Random rnd) { + @SuppressWarnings("unchecked") + public void bulkLoadIntTuples(IIndexTestContext ctx, int numTuples, Random rnd, boolean isPoint) + throws HyracksDataException { + int fieldCount = ctx.getFieldCount(); + int numKeyFields = ctx.getKeyFieldCount(); + int[] fieldValues = new int[ctx.getFieldCount()]; + int maxValue = (int) Math.ceil(Math.pow(numTuples, 1.0 / numKeyFields)); + Collection<CheckTuple> tmpCheckTuples = createCheckTuplesCollection(); + for (int i = 0; i < numTuples; i++) { + // Set keys. + setIntKeyFields(fieldValues, numKeyFields, maxValue, rnd, isPoint); + // Set values. + setIntPayloadFields(fieldValues, numKeyFields, fieldCount); + + // Set expected values. (We also use these as the pre-sorted stream + // for ordered indexes bulk loading). + ctx.insertCheckTuple(createIntCheckTuple(fieldValues, ctx.getKeyFieldCount()), tmpCheckTuples); + } + bulkLoadCheckTuples(ctx, tmpCheckTuples, false); + + // Add tmpCheckTuples to ctx check tuples for comparing searches. + for (CheckTuple checkTuple : tmpCheckTuples) { + ctx.insertCheckTuple(checkTuple, ctx.getCheckTuples()); + } + } + + protected void setIntKeyFields(int[] fieldValues, int numKeyFields, int maxValue, Random rnd, boolean isPoint) { int maxFieldPos = numKeyFields / 2; for (int j = 0; j < maxFieldPos; j++) { int k = maxFieldPos + j; int firstValue = rnd.nextInt() % maxValue; int secondValue; - do { - secondValue = rnd.nextInt() % maxValue; - } while (secondValue < firstValue); + if (isPoint) { + secondValue = firstValue; + } else { + do { + secondValue = rnd.nextInt() % maxValue; + } while (secondValue < firstValue); + } fieldValues[j] = firstValue; fieldValues[k] = secondValue; } } @Override + protected void setIntKeyFields(int[] fieldValues, int numKeyFields, int maxValue, Random rnd) { + setIntKeyFields(fieldValues, numKeyFields, maxValue, rnd, false); + } + + @Override protected void setIntPayloadFields(int[] fieldValues, int numKeyFields, int numFields) { for (int j = numKeyFields; j < numFields; j++) { fieldValues[j] = intPayloadValue++; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8ea58cd/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreePointMBRBulkLoadTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreePointMBRBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreePointMBRBulkLoadTest.java new file mode 100644 index 0000000..0df7caf --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreePointMBRBulkLoadTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.storage.am.lsm.rtree; + +import java.util.Random; + +import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory; +import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig; +import org.apache.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestContext; +import org.apache.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestHarness; +import org.apache.hyracks.storage.am.rtree.AbstractRTreeBulkLoadTest; +import org.apache.hyracks.storage.am.rtree.AbstractRTreeTestContext; +import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType; +import org.junit.After; +import org.junit.Before; + +@SuppressWarnings("rawtypes") +public class LSMRTreePointMBRBulkLoadTest extends AbstractRTreeBulkLoadTest { + + public LSMRTreePointMBRBulkLoadTest() { + super(AccessMethodTestsConfig.LSM_RTREE_TEST_RSTAR_POLICY, true); + } + + private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness(); + + @Before + public void setUp() throws HyracksDataException { + harness.setUp(); + } + + @After + public void tearDown() throws HyracksDataException { + harness.tearDown(); + } + + @Override + protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, + IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType) + throws Exception { + return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(), + harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys, + rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), + harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(), + harness.getMetadataPageManagerFactory(), true); + } + + @Override + protected Random getRandom() { + return harness.getRandom(); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8ea58cd/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java index 2bd74af..acdb268 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java @@ -79,6 +79,17 @@ public final class LSMRTreeTestContext extends AbstractRTreeTestContext { double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory, IMetadataPageManagerFactory metadataPageManagerFactory) throws Exception { + return create(ioManager, virtualBufferCaches, file, diskBufferCache, fieldSerdes, valueProviderFactories, + numKeyFields, rtreePolicyType, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, + ioOpCallbackFactory, metadataPageManagerFactory, false); + } + + public static LSMRTreeTestContext create(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, + FileReference file, IBufferCache diskBufferCache, ISerializerDeserializer[] fieldSerdes, + IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeyFields, RTreePolicyType rtreePolicyType, + double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, + ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory, + IMetadataPageManagerFactory metadataPageManagerFactory, boolean isPointMBR) throws Exception { ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes); IBinaryComparatorFactory[] rtreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields); @@ -95,7 +106,7 @@ public final class LSMRTreeTestContext extends AbstractRTreeTestContext { typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory, LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), null, btreeFields, null, - null, null, true, false, metadataPageManagerFactory); + null, null, true, isPointMBR, metadataPageManagerFactory); LSMRTreeTestContext testCtx = new LSMRTreeTestContext(fieldSerdes, lsmTree); return testCtx; }
