KYLIN-1233 Add testcases for spill and in-mem of AggregationCache
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a46a25dd Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a46a25dd Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a46a25dd Branch: refs/heads/2.x-staging Commit: a46a25dd096ad54add369e9067401229a797d11f Parents: eaed4f6 Author: lidongsjtu <don...@ebay.com> Authored: Sun Dec 27 14:34:06 2015 +0800 Committer: lidongsjtu <don...@ebay.com> Committed: Sun Dec 27 14:34:06 2015 +0800 ---------------------------------------------------------------------- .../apache/kylin/gridtable/UnitTestSupport.java | 44 +++++- .../gridtable/AggregationCacheSpillTest.java | 140 +++++++++++++++++++ 2 files changed, 182 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/a46a25dd/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java index ff71b4f..08187e9 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.BitSet; import java.util.List; +import org.apache.kylin.common.hll.HyperLogLogPlusCounter; import org.apache.kylin.common.util.DateFormat; import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.gridtable.GTInfo.Builder; @@ -44,6 +45,23 @@ public class UnitTestSupport { return info; } + public static GTInfo hllInfo() { + Builder builder = GTInfo.builder(); + builder.setCodeSystem(new GTSampleCodeSystem()); + builder.setColumns( // + DataType.getType("varchar(10)"), // + DataType.getType("varchar(10)"), // + DataType.getType("varchar(10)"), // + DataType.getType("bigint"), // + DataType.getType("decimal"), // + DataType.getType("hllc14") // + ); + builder.setPrimaryKey(setOf(0)); + builder.setColumnPreferIndex(setOf(0)); + GTInfo info = builder.build(); + return info; + } + private static Builder infoBuilder() { Builder builder = GTInfo.builder(); builder.setCodeSystem(new GTSampleCodeSystem()); @@ -81,15 +99,37 @@ public class UnitTestSupport { return result; } + public static List<GTRecord> mockupHllData(GTInfo info, int nRows) { + List<GTRecord> result = new ArrayList<GTRecord>(nRows); + int round = nRows / 10; + for (int i = 0; i < round; i++) { + String d_01_14 = datePlus("2015-01-14", i * 4); + String d_01_15 = datePlus("2015-01-15", i * 4); + String d_01_16 = datePlus("2015-01-16", i * 4); + String d_01_17 = datePlus("2015-01-17", i * 4); + result.add(newRec(info, d_01_14, "Yang", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14))); + result.add(newRec(info, d_01_14, "Luke", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14))); + result.add(newRec(info, d_01_15, "Xu", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14))); + result.add(newRec(info, d_01_15, "Dong", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14))); + result.add(newRec(info, d_01_15, "Jason", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14))); + result.add(newRec(info, d_01_16, "Mahone", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14))); + result.add(newRec(info, d_01_16, "Shaofeng", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14))); + result.add(newRec(info, d_01_16, "Qianhao", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14))); + result.add(newRec(info, d_01_16, "George", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14))); + result.add(newRec(info, d_01_17, "Kejia", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14))); + } + return result; + } + private static String datePlus(String date, int plusDays) { long millis = DateFormat.stringToMillis(date); millis += (1000L * 3600L * 24L) * plusDays; return DateFormat.formatToDateStr(millis); } - private static GTRecord newRec(GTInfo info, String date, String name, String category, LongMutable amount, BigDecimal price) { + private static GTRecord newRec(GTInfo info, Object... values) { GTRecord rec = new GTRecord(info); - return rec.setValues(date, name, category, amount, price); + return rec.setValues(values); } private static ImmutableBitSet setOf(int... values) { http://git-wip-us.apache.org/repos/asf/kylin/blob/a46a25dd/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java new file mode 100644 index 0000000..7fd5ce7 --- /dev/null +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.gridtable; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.io.IOException; +import java.math.BigDecimal; +import java.util.Iterator; +import java.util.List; + +import org.apache.kylin.common.util.ImmutableBitSet; +import org.apache.kylin.metadata.datatype.LongMutable; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.collect.Lists; + +/** + * Created by dongli on 12/16/15. + */ +public class AggregationCacheSpillTest { + + final static int DATA_CARDINALITY = 40000; + final static int DATA_REPLICATION = 2; + + final static GTInfo INFO = UnitTestSupport.hllInfo(); + final static List<GTRecord> TEST_DATA = Lists.newArrayListWithCapacity(DATA_CARDINALITY * DATA_REPLICATION);; + + @BeforeClass + public static void beforeClass() { + System.setProperty("log4j.configuration", "kylin-log4j.properties"); + + final List<GTRecord> data = UnitTestSupport.mockupHllData(INFO, DATA_CARDINALITY); + for (int i = 0; i < DATA_REPLICATION; i++) + TEST_DATA.addAll(data); + } + + @Test + public void testAggregationCacheSpill() throws IOException { + IGTScanner inputScanner = new IGTScanner() { + @Override + public GTInfo getInfo() { + return INFO; + } + + @Override + public int getScannedRowCount() { + throw new UnsupportedOperationException(); + } + + @Override + public void close() throws IOException { + } + + @Override + public Iterator<GTRecord> iterator() { + return TEST_DATA.iterator(); + } + }; + + GTScanRequest scanRequest = new GTScanRequest(INFO, null, new ImmutableBitSet(0, 3), new ImmutableBitSet(0, 3), new ImmutableBitSet(3, 6), new String[] { "SUM", "SUM", "COUNT_DISTINCT" }, null, true); + scanRequest.setAggrCacheGB(0.5); // 500 MB + + GTAggregateScanner scanner = new GTAggregateScanner(inputScanner, scanRequest); + + int count = 0; + for (GTRecord record : scanner) { + assertNotNull(record); + Object[] returnRecord = record.getValues(); + assertEquals(20, ((LongMutable) returnRecord[3]).get()); + assertEquals(21, ((BigDecimal) returnRecord[4]).longValue()); + count++; + + System.out.println(record); + } + assertEquals(DATA_CARDINALITY, count); + scanner.close(); + } + + @Test + public void testAggregationCacheInMem() throws IOException { + IGTScanner inputScanner = new IGTScanner() { + @Override + public GTInfo getInfo() { + return INFO; + } + + @Override + public int getScannedRowCount() { + throw new UnsupportedOperationException(); + } + + @Override + public void close() throws IOException { + } + + @Override + public Iterator<GTRecord> iterator() { + return TEST_DATA.iterator(); + } + }; + + // all-in-mem testcase + GTScanRequest scanRequest = new GTScanRequest(INFO, null, new ImmutableBitSet(0, 3), new ImmutableBitSet(1, 3), new ImmutableBitSet(3, 6), new String[] { "SUM", "SUM", "COUNT_DISTINCT" }, null, true); + scanRequest.setAggrCacheGB(0.5); // 500 MB + + GTAggregateScanner scanner = new GTAggregateScanner(inputScanner, scanRequest); + + int count = 0; + for (GTRecord record : scanner) { + assertNotNull(record); + Object[] returnRecord = record.getValues(); + assertEquals(80000, ((LongMutable) returnRecord[3]).get()); + assertEquals(84000, ((BigDecimal) returnRecord[4]).longValue()); + count++; + + System.out.println(record); + } + assertEquals(10, count); + scanner.close(); + } +} \ No newline at end of file
