Repository: incubator-kylin Updated Branches: refs/heads/0.8 ccdc4155e -> 993e0645e
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java deleted file mode 100644 index b3d77de..0000000 --- a/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java +++ /dev/null @@ -1,112 +0,0 @@ -package org.apache.kylin.storage.gridtable.memstore; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.kylin.common.util.ImmutableBitSet; -import org.apache.kylin.storage.gridtable.GTInfo; -import org.apache.kylin.storage.gridtable.GTRecord; -import org.apache.kylin.storage.gridtable.GTRowBlock; -import org.apache.kylin.storage.gridtable.GTScanRequest; -import org.apache.kylin.storage.gridtable.IGTStore; - -public class GTSimpleMemStore implements IGTStore { - - final GTInfo info; - final List<GTRowBlock> rowBlockList; - - public GTSimpleMemStore(GTInfo info) { - this.info = info; - this.rowBlockList = new ArrayList<GTRowBlock>(); - - if (info.isShardingEnabled()) - throw new UnsupportedOperationException(); - } - - @Override - public GTInfo getInfo() { - return info; - } - - public long memoryUsage() { - if (rowBlockList.size() == 0) { - return 0; - } else { - return rowBlockList.get(0).exportLength() * Long.valueOf(rowBlockList.size()); - } - } - - @Override - public IGTStoreWriter rebuild(int shard) { - rowBlockList.clear(); - return new Writer(rowBlockList); - } - - @Override - public IGTStoreWriter append(int shard, GTRowBlock.Writer fillLast) { - if (rowBlockList.size() > 0) { - GTRowBlock last = rowBlockList.get(rowBlockList.size() - 1); - fillLast.copyFrom(last); - } - return new Writer(rowBlockList); - } - - private static class Writer implements IGTStoreWriter { - - private final List<GTRowBlock> rowBlockList; - - Writer(List<GTRowBlock> rowBlockList) { - this.rowBlockList = rowBlockList; - } - @Override - public void close() throws IOException { - } - - @Override - public void write(GTRowBlock block) throws IOException { - GTRowBlock copy = block.copy(); - int id = block.getSequenceId(); - if (id < rowBlockList.size()) { - rowBlockList.set(id, copy); - } else { - assert id == rowBlockList.size(); - rowBlockList.add(copy); - } - } - } - - @Override - public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, ImmutableBitSet selectedColBlocks, GTScanRequest additionalPushDown) { - - return new IGTStoreScanner() { - Iterator<GTRowBlock> it = rowBlockList.iterator(); - - @Override - public boolean hasNext() { - return it.hasNext(); - } - - @Override - public GTRowBlock next() { - return it.next(); - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - @Override - public void close() throws IOException { - } - }; - } - - public void drop() throws IOException { - //will there be any concurrent issue? If yes, ArrayList should be replaced - rowBlockList.clear(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java index 63623ca..1a804cd 100644 --- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java +++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java @@ -36,6 +36,7 @@ import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.tuple.ITupleIterator; import org.apache.kylin.storage.StorageContext; import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler; +import org.apache.kylin.storage.hbase.steps.RowValueDecoder; import org.apache.kylin.storage.tuple.Tuple; import org.apache.kylin.storage.tuple.TupleInfo; import org.slf4j.Logger; http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageQuery.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageQuery.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageQuery.java index bd5f21a..4c3920b 100644 --- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageQuery.java +++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageQuery.java @@ -50,6 +50,7 @@ import org.apache.kylin.metadata.tuple.ITupleIterator; import org.apache.kylin.storage.ICachableStorageQuery; import org.apache.kylin.storage.StorageContext; import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler; +import org.apache.kylin.storage.hbase.steps.RowValueDecoder; import org.apache.kylin.storage.tuple.TupleInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/storage/src/main/java/org/apache/kylin/storage/hbase/CubeTupleConverter.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeTupleConverter.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeTupleConverter.java index 8d0af58..08c3784 100644 --- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeTupleConverter.java +++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeTupleConverter.java @@ -17,6 +17,7 @@ import org.apache.kylin.dict.lookup.LookupStringTable; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.storage.hbase.steps.RowValueDecoder; import org.apache.kylin.storage.tuple.Tuple; import org.apache.kylin.storage.tuple.TupleInfo; http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java index 5e6d9a0..96e4df6 100644 --- a/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java +++ b/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java @@ -30,6 +30,7 @@ import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.tuple.ITuple; import org.apache.kylin.metadata.tuple.ITupleIterator; import org.apache.kylin.storage.StorageContext; +import org.apache.kylin.storage.hbase.steps.RowValueDecoder; import org.apache.kylin.storage.tuple.TupleInfo; import com.google.common.collect.Lists; http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverAggregators.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverAggregators.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverAggregators.java index 8e0e8f7..a81dcfc 100644 --- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverAggregators.java +++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverAggregators.java @@ -24,8 +24,8 @@ import java.util.Collection; import java.util.Comparator; import java.util.List; -import org.apache.kylin.storage.hbase.RowValueDecoder; import org.apache.kylin.storage.hbase.coprocessor.CoprocessorConstants; +import org.apache.kylin.storage.hbase.steps.RowValueDecoder; import org.apache.hadoop.hbase.Cell; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.common.util.BytesSerializer; http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java index 3bbf43a..2aeca8f 100644 --- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java +++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java @@ -35,11 +35,11 @@ import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.storage.StorageContext; import org.apache.kylin.storage.hbase.RegionScannerAdapter; import org.apache.kylin.storage.hbase.ResultScannerAdapter; -import org.apache.kylin.storage.hbase.RowValueDecoder; import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter; import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector; import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType; import org.apache.kylin.storage.hbase.coprocessor.FilterDecorator; +import org.apache.kylin.storage.hbase.steps.RowValueDecoder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java deleted file mode 100644 index c6b1a18..0000000 --- a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.storage.tuple; - -import java.math.BigDecimal; -import java.util.Date; -import java.util.List; - -import net.sf.ehcache.pool.sizeof.annotations.IgnoreSizeOf; - -import org.apache.kylin.common.util.DateFormat; -import org.apache.kylin.metadata.measure.DoubleMutable; -import org.apache.kylin.metadata.measure.LongMutable; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.tuple.ITuple; - -/** - * @author xjiang - */ -public class Tuple implements ITuple { - - @IgnoreSizeOf - private final TupleInfo info; - private final Object[] values; - - public Tuple(TupleInfo info) { - this.info = info; - this.values = new Object[info.size()]; - } - - public List<String> getAllFields() { - return info.getAllFields(); - } - - public List<TblColRef> getAllColumns() { - return info.getAllColumns(); - } - - public Object[] getAllValues() { - return values; - } - - @Override - public ITuple makeCopy() { - Tuple ret = new Tuple(this.info); - for (int i = 0; i < this.values.length; ++i) { - ret.values[i] = this.values[i]; - } - return ret; - } - - public TupleInfo getInfo() { - return info; - } - - public String getFieldName(TblColRef col) { - return info.getFieldName(col); - } - - public TblColRef getFieldColumn(String fieldName) { - return info.getColumn(fieldName); - } - - public Object getValue(String fieldName) { - int index = info.getFieldIndex(fieldName); - return values[index]; - } - - public Object getValue(TblColRef col) { - int index = info.getColumnIndex(col); - return values[index]; - } - - public String getDataTypeName(int idx) { - return info.getDataTypeName(idx); - } - - public void setDimensionValue(String fieldName, String fieldValue) { - setDimensionValue(info.getFieldIndex(fieldName), fieldValue); - } - - public void setDimensionValue(int idx, String fieldValue) { - Object objectValue = convertOptiqCellValue(fieldValue, getDataTypeName(idx)); - values[idx] = objectValue; - } - - public void setMeasureValue(String fieldName, Object fieldValue) { - setMeasureValue(info.getFieldIndex(fieldName), fieldValue); - } - - public void setMeasureValue(int idx, Object fieldValue) { - fieldValue = convertWritableToJava(fieldValue); - - String dataType = getDataTypeName(idx); - // special handling for BigDecimal, allow double be aggregated as - // BigDecimal during cube build for best precision - if ("double".equals(dataType) && fieldValue instanceof BigDecimal) { - fieldValue = ((BigDecimal) fieldValue).doubleValue(); - } else if ("integer".equals(dataType) && !(fieldValue instanceof Integer)) { - fieldValue = ((Number) fieldValue).intValue(); - } else if ("float".equals(dataType) && fieldValue instanceof BigDecimal) { - fieldValue = ((BigDecimal) fieldValue).floatValue(); - } - values[idx] = fieldValue; - } - - private Object convertWritableToJava(Object o) { - if (o instanceof LongMutable) - o = ((LongMutable) o).get(); - else if (o instanceof DoubleMutable) - o = ((DoubleMutable) o).get(); - return o; - } - - public boolean hasColumn(TblColRef column) { - return info.hasColumn(column); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - for (String field : info.getAllFields()) { - sb.append(field); - sb.append("="); - sb.append(getValue(field)); - sb.append(","); - } - return sb.toString(); - } - - public static long epicDaysToMillis(int days) { - return 1L * days * (1000 * 3600 * 24); - } - - public static int dateToEpicDays(String strValue) { - Date dateValue = DateFormat.stringToDate(strValue); // NOTE: forces GMT timezone - long millis = dateValue.getTime(); - return (int) (millis / (1000 * 3600 * 24)); - } - - public static long getTs(ITuple row, TblColRef partitionCol) { - //ts column type differentiate - if (partitionCol.getDatatype().equals("date")) { - return Tuple.epicDaysToMillis(Integer.valueOf(row.getValue(partitionCol).toString())); - } else { - return Long.valueOf(row.getValue(partitionCol).toString()); - } - } - - public static Object convertOptiqCellValue(String strValue, String dataTypeName) { - if (strValue == null) - return null; - - if ((strValue.equals("") || strValue.equals("\\N")) && !dataTypeName.equals("string")) - return null; - - // TODO use data type enum instead of string comparison - if ("date".equals(dataTypeName)) { - // convert epoch time - return dateToEpicDays(strValue);// Optiq expects Integer instead of Long. by honma - } else if ("timestamp".equals(dataTypeName) || "datetime".equals(dataTypeName)) { - return Long.valueOf(DateFormat.stringToMillis(strValue)); - } else if ("tinyint".equals(dataTypeName)) { - return Byte.valueOf(strValue); - } else if ("short".equals(dataTypeName) || "smallint".equals(dataTypeName)) { - return Short.valueOf(strValue); - } else if ("integer".equals(dataTypeName)) { - return Integer.valueOf(strValue); - } else if ("long".equals(dataTypeName) || "bigint".equals(dataTypeName)) { - return Long.valueOf(strValue); - } else if ("double".equals(dataTypeName)) { - return Double.valueOf(strValue); - } else if ("decimal".equals(dataTypeName)) { - return new BigDecimal(strValue); - } else if ("float".equals(dataTypeName)){ - return Float.valueOf(strValue); - } else { - return strValue; - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/storage/src/main/java/org/apache/kylin/storage/tuple/TupleInfo.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/tuple/TupleInfo.java b/storage/src/main/java/org/apache/kylin/storage/tuple/TupleInfo.java deleted file mode 100644 index 735cc64..0000000 --- a/storage/src/main/java/org/apache/kylin/storage/tuple/TupleInfo.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.storage.tuple; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.kylin.metadata.model.TblColRef; - -/** - * - * @author xjiang - * - */ -public class TupleInfo { - - private final Map<String, Integer> fieldMap; - private final Map<TblColRef, Integer> columnMap; - - private final List<String> fields; - private final List<TblColRef> columns; - private final List<String> dataTypeNames; - - public TupleInfo() { - fieldMap = new HashMap<String, Integer>(); - columnMap = new HashMap<TblColRef, Integer>(); - fields = new ArrayList<String>(); - columns = new ArrayList<TblColRef>(); - dataTypeNames = new ArrayList<String>(); - } - - public TblColRef getColumn(String fieldName) { - int idx = getFieldIndex(fieldName); - return columns.get(idx); - } - - public int getColumnIndex(TblColRef col) { - return columnMap.get(col); - } - - public String getDataTypeName(int index) { - return dataTypeNames.get(index); - } - - public int getFieldIndex(String fieldName) { - return fieldMap.get(fieldName); - } - - public boolean hasField(String fieldName) { - return fieldMap.containsKey(fieldName); - } - - public String getFieldName(TblColRef col) { - int idx = columnMap.get(col); - return fields.get(idx); - } - - public boolean hasColumn(TblColRef col) { - return columnMap.containsKey(col); - } - - public void setField(String fieldName, TblColRef col, int index) { - fieldMap.put(fieldName, index); - - if (col != null) - columnMap.put(col, index); - - if (fields.size() > index) - fields.set(index, fieldName); - else - fields.add(index, fieldName); - - if (columns.size() > index) - columns.set(index, col); - else - columns.add(index, col); - - if (dataTypeNames.size() > index) - dataTypeNames.set(index, col.getType().getName()); - else - dataTypeNames.add(index, col.getType().getName()); - } - - public List<String> getAllFields() { - return fields; - } - - public List<TblColRef> getAllColumns() { - return columns; - } - - public int size() { - return fields.size(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/storage/src/test/java/org/apache/kylin/storage/gridtable/AggregationCacheMemSizeTest.java ---------------------------------------------------------------------- diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/AggregationCacheMemSizeTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/AggregationCacheMemSizeTest.java deleted file mode 100644 index 649fd5a..0000000 --- a/storage/src/test/java/org/apache/kylin/storage/gridtable/AggregationCacheMemSizeTest.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.storage.gridtable; - -import java.math.BigDecimal; -import java.util.Comparator; -import java.util.Random; -import java.util.SortedMap; -import java.util.TreeMap; - -import org.apache.kylin.common.hll.HyperLogLogPlusCounter; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.metadata.measure.BigDecimalSumAggregator; -import org.apache.kylin.metadata.measure.DoubleSumAggregator; -import org.apache.kylin.metadata.measure.DoubleMutable; -import org.apache.kylin.metadata.measure.HLLCAggregator; -import org.apache.kylin.metadata.measure.LongSumAggregator; -import org.apache.kylin.metadata.measure.LongMutable; -import org.apache.kylin.metadata.measure.MeasureAggregator; -import org.junit.Test; - -public class AggregationCacheMemSizeTest { - - public static final int NUM_OF_OBJS = 1000000 / 2; - - interface CreateAnObject { - Object create(); - } - - @Test - public void testHLLCAggregatorSize() throws InterruptedException { - int est = estimateObjectSize(new CreateAnObject() { - @Override - public Object create() { - HLLCAggregator aggr = new HLLCAggregator(10); - aggr.aggregate(new HyperLogLogPlusCounter(10)); - return aggr; - } - }); - System.out.println("HLLC: " + est); - } - - @Test - public void testBigDecimalAggregatorSize() throws InterruptedException { - int est = estimateObjectSize(new CreateAnObject() { - @Override - public Object create() { - return newBigDecimalAggr(); - } - - }); - System.out.println("BigDecimal: " + est); - } - - private BigDecimalSumAggregator newBigDecimalAggr() { - BigDecimalSumAggregator aggr = new BigDecimalSumAggregator(); - aggr.aggregate(new BigDecimal("12345678901234567890.123456789")); - return aggr; - } - - @Test - public void testLongAggregatorSize() throws InterruptedException { - int est = estimateObjectSize(new CreateAnObject() { - @Override - public Object create() { - return newLongAggr(); - } - }); - System.out.println("Long: " + est); - } - - private LongSumAggregator newLongAggr() { - LongSumAggregator aggr = new LongSumAggregator(); - aggr.aggregate(new LongMutable(10)); - return aggr; - } - - @Test - public void testDoubleAggregatorSize() throws InterruptedException { - int est = estimateObjectSize(new CreateAnObject() { - @Override - public Object create() { - return newDoubleAggr(); - } - }); - System.out.println("Double: " + est); - } - - private DoubleSumAggregator newDoubleAggr() { - DoubleSumAggregator aggr = new DoubleSumAggregator(); - aggr.aggregate(new DoubleMutable(10)); - return aggr; - } - - @Test - public void testByteArraySize() throws InterruptedException { - int est = estimateObjectSize(new CreateAnObject() { - @Override - public Object create() { - return new byte[10]; - } - }); - System.out.println("byte[10]: " + est); - } - - @Test - public void testAggregatorArraySize() throws InterruptedException { - int est = estimateObjectSize(new CreateAnObject() { - @Override - public Object create() { - return new MeasureAggregator[7]; - } - }); - System.out.println("MeasureAggregator[7]: " + est); - } - - @Test - public void testTreeMapSize() throws InterruptedException { - final SortedMap<byte[], Object> map = new TreeMap<byte[], Object>(new Comparator<byte[]>() { - @Override - public int compare(byte[] o1, byte[] o2) { - return Bytes.compareTo(o1, o2); - } - }); - final Random rand = new Random(); - int est = estimateObjectSize(new CreateAnObject() { - @Override - public Object create() { - byte[] key = new byte[10]; - rand.nextBytes(key); - map.put(key, null); - return null; - } - }); - System.out.println("TreeMap entry: " + (est - 20)); // -20 is to exclude byte[10] - } - - @Test - public void testAggregationCacheSize() throws InterruptedException { - final SortedMap<byte[], Object> map = new TreeMap<byte[], Object>(new Comparator<byte[]>() { - @Override - public int compare(byte[] o1, byte[] o2) { - return Bytes.compareTo(o1, o2); - } - }); - final Random rand = new Random(); - - long bytesBefore = memLeft(); - byte[] key = null; - MeasureAggregator<?>[] aggrs = null; - for (int i = 0; i < NUM_OF_OBJS; i++) { - key = new byte[10]; - rand.nextBytes(key); - aggrs = new MeasureAggregator[4]; - aggrs[0] = newBigDecimalAggr(); - aggrs[1] = newLongAggr(); - aggrs[2] = newDoubleAggr(); - aggrs[3] = newDoubleAggr(); - map.put(key, aggrs); - } - - long bytesAfter = memLeft(); - - long mapActualSize = bytesBefore - bytesAfter; - long mapExpectSize = GTAggregateScanner.estimateSizeOfAggrCache(key, aggrs, map.size()); - System.out.println("Actual cache size: " + mapActualSize); - System.out.println("Expect cache size: " + mapExpectSize); - } - - private int estimateObjectSize(CreateAnObject factory) throws InterruptedException { - Object[] hold = new Object[NUM_OF_OBJS]; - long bytesBefore = memLeft(); - - for (int i = 0; i < hold.length; i++) { - hold[i] = factory.create(); - } - - long bytesAfter = memLeft(); - return (int) ((bytesBefore - bytesAfter) / hold.length); - } - - private long memLeft() throws InterruptedException { - Runtime.getRuntime().gc(); - Thread.sleep(500); - return getSystemAvailBytes(); - } - - private long getSystemAvailBytes() { - Runtime runtime = Runtime.getRuntime(); - long totalMemory = runtime.totalMemory(); // current heap allocated to the VM process - long freeMemory = runtime.freeMemory(); // out of the current heap, how much is free - long maxMemory = runtime.maxMemory(); // Max heap VM can use e.g. Xmx setting - long usedMemory = totalMemory - freeMemory; // how much of the current heap the VM is using - long availableMemory = maxMemory - usedMemory; // available memory i.e. Maximum heap size minus the current amount used - return availableMemory; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java ---------------------------------------------------------------------- diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java deleted file mode 100644 index 6092956..0000000 --- a/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.storage.gridtable; - -import static org.junit.Assert.*; - -import java.io.IOException; -import java.math.BigDecimal; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.BitSet; -import java.util.List; -import java.util.Map; - -import org.apache.kylin.common.util.ByteArray; -import org.apache.kylin.common.util.ImmutableBitSet; -import org.apache.kylin.dict.Dictionary; -import org.apache.kylin.dict.NumberDictionaryBuilder; -import org.apache.kylin.dict.StringBytesConverter; -import org.apache.kylin.dict.TrieDictionaryBuilder; -import org.apache.kylin.metadata.filter.ColumnTupleFilter; -import org.apache.kylin.metadata.filter.CompareTupleFilter; -import org.apache.kylin.metadata.filter.ConstantTupleFilter; -import org.apache.kylin.metadata.filter.ExtractTupleFilter; -import org.apache.kylin.metadata.filter.LogicalTupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; -import org.apache.kylin.metadata.measure.LongMutable; -import org.apache.kylin.metadata.model.ColumnDesc; -import org.apache.kylin.metadata.model.DataType; -import org.apache.kylin.metadata.model.TableDesc; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.storage.cube.CubeCodeSystem; -import org.apache.kylin.storage.gridtable.GTInfo.Builder; -import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore; -import org.junit.Test; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -public class DictGridTableTest { - - @Test - public void test() throws IOException { - GridTable table = newTestTable(); - verifyScanRangePlanner(table); - verifyFirstRow(table); - verifyScanWithUnevaluatableFilter(table); - verifyScanWithEvaluatableFilter(table); - verifyConvertFilterConstants1(table); - verifyConvertFilterConstants2(table); - verifyConvertFilterConstants3(table); - verifyConvertFilterConstants4(table); - } - - private void verifyScanRangePlanner(GridTable table) { - GTInfo info = table.getInfo(); - GTScanRangePlanner planner = new GTScanRangePlanner(info); - - CompareTupleFilter timeComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14")); - CompareTupleFilter timeComp2 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-13")); - CompareTupleFilter timeComp3 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-15")); - CompareTupleFilter timeComp4 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "2015-01-15")); - CompareTupleFilter ageComp1 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "10")); - CompareTupleFilter ageComp2 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "20")); - CompareTupleFilter ageComp3 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "30")); - CompareTupleFilter ageComp4 = compare(info.colRef(1), FilterOperatorEnum.NEQ, enc(info, 1, "30")); - - // flatten or-and & hbase fuzzy value - { - LogicalTupleFilter filter = and(timeComp1, or(ageComp1, ageComp2)); - List<GTScanRange> r = planner.planScanRanges(filter); - assertEquals(1, r.size()); - assertEquals("[1421193600000, 10]-[null, null]", r.get(0).toString()); - assertEquals("[[10], [20]]", r.get(0).hbaseFuzzyKeys.toString()); - } - - // pre-evaluate ever false - { - LogicalTupleFilter filter = and(timeComp1, timeComp2); - List<GTScanRange> r = planner.planScanRanges(filter); - assertEquals(0, r.size()); - } - - // pre-evaluate ever true - { - LogicalTupleFilter filter = or(timeComp1, ageComp4); - List<GTScanRange> r = planner.planScanRanges(filter); - assertEquals("[[null, null]-[null, null]]", r.toString()); - } - - // merge overlap range - { - LogicalTupleFilter filter = or(timeComp1, timeComp3); - List<GTScanRange> r = planner.planScanRanges(filter); - assertEquals("[[null, null]-[null, null]]", r.toString()); - } - - // merge too many ranges - { - LogicalTupleFilter filter = or(and(timeComp4, ageComp1), and(timeComp4, ageComp2), and(timeComp4, ageComp3)); - List<GTScanRange> r = planner.planScanRanges(filter); - assertEquals(3, r.size()); - assertEquals("[1421280000000, 10]-[1421280000000, 10]", r.get(0).toString()); - assertEquals("[1421280000000, 20]-[1421280000000, 20]", r.get(1).toString()); - assertEquals("[1421280000000, 30]-[1421280000000, 30]", r.get(2).toString()); - List<GTScanRange> r2 = planner.planScanRanges(filter, 2); - assertEquals("[[1421280000000, 10]-[1421280000000, 30]]", r2.toString()); - } - } - - private void verifyFirstRow(GridTable table) throws IOException { - doScanAndVerify(table, new GTScanRequest(table.getInfo()), "[1421193600000, 30, Yang, 10, 10.5]"); - } - - private void verifyScanWithUnevaluatableFilter(GridTable table) throws IOException { - GTInfo info = table.getInfo(); - - CompareTupleFilter fComp = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14")); - ExtractTupleFilter fUnevaluatable = unevaluatable(info.colRef(1)); - LogicalTupleFilter fNotPlusUnevaluatable = not(unevaluatable(info.colRef(1))); - LogicalTupleFilter filter = and(fComp, fUnevaluatable, fNotPlusUnevaluatable); - - GTScanRequest req = new GTScanRequest(info, null, setOf(0), setOf(3), new String[] { "sum" }, filter); - - // note the unEvaluatable column 1 in filter is added to group by - assertEquals("GTScanRequest [range=[null, null]-[null, null], columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], [null], [null]], aggrGroupBy={0, 1}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString()); - - doScanAndVerify(table, req, "[1421280000000, 20, null, 20, null]"); - } - - private void verifyScanWithEvaluatableFilter(GridTable table) throws IOException { - GTInfo info = table.getInfo(); - - CompareTupleFilter fComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14")); - CompareTupleFilter fComp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10")); - LogicalTupleFilter filter = and(fComp1, fComp2); - - GTScanRequest req = new GTScanRequest(info, null, setOf(0), setOf(3), new String[] { "sum" }, filter); - - // note the evaluatable column 1 in filter is added to returned columns but not in group by - assertEquals("GTScanRequest [range=[null, null]-[null, null], columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 GT [\\x00]], aggrGroupBy={0}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString()); - - doScanAndVerify(table, req, "[1421280000000, 20, null, 30, null]", "[1421366400000, 20, null, 40, null]"); - } - - private void verifyConvertFilterConstants1(GridTable table) { - GTInfo info = table.getInfo(); - - TableDesc extTable = TableDesc.mockup("ext"); - TblColRef extColA = new TblColRef(ColumnDesc.mockup(extTable, 1, "A", "timestamp")); - TblColRef extColB = new TblColRef(ColumnDesc.mockup(extTable, 2, "B", "integer")); - - CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14"); - CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.EQ, "10"); - LogicalTupleFilter filter = and(fComp1, fComp2); - - List<TblColRef> colMapping = Lists.newArrayList(); - colMapping.add(extColA); - colMapping.add(extColB); - - TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); - assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 EQ [\\x00]]", newFilter.toString()); - } - - private void verifyConvertFilterConstants2(GridTable table) { - GTInfo info = table.getInfo(); - - TableDesc extTable = TableDesc.mockup("ext"); - TblColRef extColA = new TblColRef(ColumnDesc.mockup(extTable, 1, "A", "timestamp")); - TblColRef extColB = new TblColRef(ColumnDesc.mockup(extTable, 2, "B", "integer")); - - CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14"); - CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.LT, "9"); - LogicalTupleFilter filter = and(fComp1, fComp2); - - List<TblColRef> colMapping = Lists.newArrayList(); - colMapping.add(extColA); - colMapping.add(extColB); - - // $1<"9" round up to $1<"10" - TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); - assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 LT [\\x00]]", newFilter.toString()); - } - - private void verifyConvertFilterConstants3(GridTable table) { - GTInfo info = table.getInfo(); - - TableDesc extTable = TableDesc.mockup("ext"); - TblColRef extColA = new TblColRef(ColumnDesc.mockup(extTable, 1, "A", "timestamp")); - TblColRef extColB = new TblColRef(ColumnDesc.mockup(extTable, 2, "B", "integer")); - - CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14"); - CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.LTE, "9"); - LogicalTupleFilter filter = and(fComp1, fComp2); - - List<TblColRef> colMapping = Lists.newArrayList(); - colMapping.add(extColA); - colMapping.add(extColB); - - // $1<="9" round down to FALSE - TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); - assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], []]", newFilter.toString()); - } - - private void verifyConvertFilterConstants4(GridTable table) { - GTInfo info = table.getInfo(); - - TableDesc extTable = TableDesc.mockup("ext"); - TblColRef extColA = new TblColRef(ColumnDesc.mockup(extTable, 1, "A", "timestamp")); - TblColRef extColB = new TblColRef(ColumnDesc.mockup(extTable, 2, "B", "integer")); - - CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14"); - CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.IN, "9", "10", "15"); - LogicalTupleFilter filter = and(fComp1, fComp2); - - List<TblColRef> colMapping = Lists.newArrayList(); - colMapping.add(extColA); - colMapping.add(extColB); - - // $1 in ("9", "10", "15") has only "10" left - TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); - assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 IN [\\x00]]", newFilter.toString()); - } - - private void doScanAndVerify(GridTable table, GTScanRequest req, String... verifyRows) throws IOException { - System.out.println(req); - IGTScanner scanner = table.scan(req); - int i = 0; - for (GTRecord r : scanner) { - System.out.println(r); - if (verifyRows != null && i < verifyRows.length) { - assertEquals(verifyRows[i], r.toString()); - } - i++; - } - scanner.close(); - } - - private Object enc(GTInfo info, int col, String value) { - ByteBuffer buf = ByteBuffer.allocate(info.getMaxColumnLength()); - info.codeSystem.encodeColumnValue(col, value, buf); - return ByteArray.copyOf(buf.array(), buf.arrayOffset(), buf.position()); - } - - private ExtractTupleFilter unevaluatable(TblColRef col) { - ExtractTupleFilter r = new ExtractTupleFilter(FilterOperatorEnum.EXTRACT); - r.addChild(new ColumnTupleFilter(col)); - return r; - } - - private CompareTupleFilter compare(TblColRef col, FilterOperatorEnum op, Object... value) { - CompareTupleFilter result = new CompareTupleFilter(op); - result.addChild(new ColumnTupleFilter(col)); - result.addChild(new ConstantTupleFilter(Arrays.asList(value))); - return result; - } - - private LogicalTupleFilter and(TupleFilter... children) { - return logic(FilterOperatorEnum.AND, children); - } - - private LogicalTupleFilter or(TupleFilter... children) { - return logic(FilterOperatorEnum.OR, children); - } - - private LogicalTupleFilter not(TupleFilter child) { - return logic(FilterOperatorEnum.NOT, child); - } - - private LogicalTupleFilter logic(FilterOperatorEnum op, TupleFilter... children) { - LogicalTupleFilter result = new LogicalTupleFilter(op); - for (TupleFilter c : children) { - result.addChild(c); - } - return result; - } - - static GridTable newTestTable() throws IOException { - GTInfo info = newInfo(); - GTSimpleMemStore store = new GTSimpleMemStore(info); - GridTable table = new GridTable(info, store); - - GTRecord r = new GTRecord(table.getInfo()); - GTBuilder builder = table.rebuild(); - - builder.write(r.setValues("2015-01-14", "30", "Yang", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-14", "30", "Luke", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-15", "20", "Dong", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-15", "20", "Jason", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-15", "30", "Xu", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-16", "30", "George", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongMutable(10), new BigDecimal("10.5"))); - builder.close(); - - return table; - } - - static GTInfo newInfo() { - Builder builder = GTInfo.builder(); - builder.setCodeSystem(newDictCodeSystem()); - builder.setColumns( // - DataType.getInstance("timestamp"), // - DataType.getInstance("integer"), // - DataType.getInstance("varchar(10)"), // - DataType.getInstance("bigint"), // - DataType.getInstance("decimal") // - ); - builder.setPrimaryKey(setOf(0, 1)); - builder.setColumnPreferIndex(setOf(0)); - builder.enableColumnBlock(new ImmutableBitSet[] { setOf(0, 1), setOf(2), setOf(3, 4) }); - builder.enableRowBlock(4); - GTInfo info = builder.build(); - return info; - } - - @SuppressWarnings("rawtypes") - private static CubeCodeSystem newDictCodeSystem() { - Map<Integer, Dictionary> dictionaryMap = Maps.newHashMap(); - dictionaryMap.put(1, newDictionaryOfInteger()); - dictionaryMap.put(2, newDictionaryOfString()); - return new CubeCodeSystem(dictionaryMap); - } - - @SuppressWarnings("rawtypes") - private static Dictionary newDictionaryOfString() { - TrieDictionaryBuilder<String> builder = new TrieDictionaryBuilder<>(new StringBytesConverter()); - builder.addValue("Dong"); - builder.addValue("George"); - builder.addValue("Jason"); - builder.addValue("Kejia"); - builder.addValue("Luke"); - builder.addValue("Mahone"); - builder.addValue("Qianhao"); - builder.addValue("Shaofeng"); - builder.addValue("Xu"); - builder.addValue("Yang"); - return builder.build(0); - } - - @SuppressWarnings("rawtypes") - private static Dictionary newDictionaryOfInteger() { - NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<>(new StringBytesConverter()); - builder.addValue("10"); - builder.addValue("20"); - builder.addValue("30"); - builder.addValue("40"); - builder.addValue("50"); - builder.addValue("60"); - builder.addValue("70"); - builder.addValue("80"); - builder.addValue("90"); - builder.addValue("100"); - return builder.build(0); - } - - private static ImmutableBitSet setOf(int... values) { - BitSet set = new BitSet(); - for (int i : values) - set.set(i); - return new ImmutableBitSet(set); - } -} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleGridTableTest.java ---------------------------------------------------------------------- diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleGridTableTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleGridTableTest.java deleted file mode 100644 index be0d13e..0000000 --- a/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleGridTableTest.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.storage.gridtable; - -import static org.junit.Assert.*; - -import java.io.IOException; -import java.math.BigDecimal; -import java.util.BitSet; -import java.util.List; - -import org.apache.kylin.common.util.ImmutableBitSet; -import org.apache.kylin.metadata.measure.LongMutable; -import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore; -import org.junit.Test; - -public class SimpleGridTableTest { - - @Test - public void testBasics() throws IOException { - GTInfo info = UnitTestSupport.basicInfo(); - GTSimpleMemStore store = new GTSimpleMemStore(info); - GridTable table = new GridTable(info, store); - - GTBuilder builder = rebuild(table); - IGTScanner scanner = scan(table); - assertEquals(builder.getWrittenRowBlockCount(), scanner.getScannedRowBlockCount()); - assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount()); - } - - @Test - public void testAdvanced() throws IOException { - GTInfo info = UnitTestSupport.advancedInfo(); - GTSimpleMemStore store = new GTSimpleMemStore(info); - GridTable table = new GridTable(info, store); - - GTBuilder builder = rebuild(table); - IGTScanner scanner = scan(table); - assertEquals(builder.getWrittenRowBlockCount(), scanner.getScannedRowBlockCount()); - assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount()); - } - - @Test - public void testAggregate() throws IOException { - GTInfo info = UnitTestSupport.advancedInfo(); - GTSimpleMemStore store = new GTSimpleMemStore(info); - GridTable table = new GridTable(info, store); - - GTBuilder builder = rebuild(table); - IGTScanner scanner = scanAndAggregate(table); - assertEquals(builder.getWrittenRowBlockCount(), scanner.getScannedRowBlockCount()); - assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount()); - } - - @Test - public void testAppend() throws IOException { - GTInfo info = UnitTestSupport.advancedInfo(); - GTSimpleMemStore store = new GTSimpleMemStore(info); - GridTable table = new GridTable(info, store); - - rebuildViaAppend(table); - IGTScanner scanner = scan(table); - assertEquals(3, scanner.getScannedRowBlockCount()); - assertEquals(10, scanner.getScannedRowCount()); - } - - private IGTScanner scan(GridTable table) throws IOException { - GTScanRequest req = new GTScanRequest(table.getInfo()); - IGTScanner scanner = table.scan(req); - for (GTRecord r : scanner) { - Object[] v = r.getValues(); - assertTrue(((String) v[0]).startsWith("2015-")); - assertTrue(((String) v[2]).equals("Food")); - assertTrue(((LongMutable) v[3]).get() == 10); - assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5); - System.out.println(r); - } - scanner.close(); - System.out.println("Scanned Row Block Count: " + scanner.getScannedRowBlockCount()); - System.out.println("Scanned Row Count: " + scanner.getScannedRowCount()); - return scanner; - } - - private IGTScanner scanAndAggregate(GridTable table) throws IOException { - GTScanRequest req = new GTScanRequest(table.getInfo(), null, setOf(0, 2), setOf(3, 4), new String[] { "count", "sum" }, null); - IGTScanner scanner = table.scan(req); - int i = 0; - for (GTRecord r : scanner) { - Object[] v = r.getValues(); - switch (i) { - case 0: - assertTrue(((LongMutable) v[3]).get() == 20); - assertTrue(((BigDecimal) v[4]).doubleValue() == 21.0); - break; - case 1: - assertTrue(((LongMutable) v[3]).get() == 30); - assertTrue(((BigDecimal) v[4]).doubleValue() == 31.5); - break; - case 2: - assertTrue(((LongMutable) v[3]).get() == 40); - assertTrue(((BigDecimal) v[4]).doubleValue() == 42.0); - break; - case 3: - assertTrue(((LongMutable) v[3]).get() == 10); - assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5); - break; - default: - fail(); - } - i++; - System.out.println(r); - } - scanner.close(); - System.out.println("Scanned Row Block Count: " + scanner.getScannedRowBlockCount()); - System.out.println("Scanned Row Count: " + scanner.getScannedRowCount()); - return scanner; - } - - static GTBuilder rebuild(GridTable table) throws IOException { - GTBuilder builder = table.rebuild(); - for (GTRecord rec : UnitTestSupport.mockupData(table.getInfo(), 10)) { - builder.write(rec); - } - builder.close(); - - System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount()); - System.out.println("Written Row Count: " + builder.getWrittenRowCount()); - return builder; - } - - static void rebuildViaAppend(GridTable table) throws IOException { - List<GTRecord> data = UnitTestSupport.mockupData(table.getInfo(), 10); - GTBuilder builder; - int i = 0; - - builder = table.append(); - builder.write(data.get(i++)); - builder.write(data.get(i++)); - builder.write(data.get(i++)); - builder.write(data.get(i++)); - builder.close(); - System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount()); - System.out.println("Written Row Count: " + builder.getWrittenRowCount()); - - builder = table.append(); - builder.write(data.get(i++)); - builder.write(data.get(i++)); - builder.write(data.get(i++)); - builder.close(); - System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount()); - System.out.println("Written Row Count: " + builder.getWrittenRowCount()); - - builder = table.append(); - builder.write(data.get(i++)); - builder.write(data.get(i++)); - builder.close(); - System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount()); - System.out.println("Written Row Count: " + builder.getWrittenRowCount()); - - builder = table.append(); - builder.write(data.get(i++)); - builder.close(); - System.out.println("Written Row Block Count: " + builder.getWrittenRowBlockCount()); - System.out.println("Written Row Count: " + builder.getWrittenRowCount()); - } - - private static ImmutableBitSet setOf(int... values) { - BitSet set = new BitSet(); - for (int i : values) - set.set(i); - return new ImmutableBitSet(set); - } -} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleInvertedIndexTest.java ---------------------------------------------------------------------- diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleInvertedIndexTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleInvertedIndexTest.java deleted file mode 100644 index a9ab61c..0000000 --- a/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleInvertedIndexTest.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.storage.gridtable; - -import static org.junit.Assert.*; -import it.uniroma3.mat.extendedset.intset.ConciseSet; - -import java.math.BigDecimal; -import java.nio.ByteBuffer; -import java.util.ArrayList; - -import org.apache.kylin.common.util.ByteArray; -import org.apache.kylin.metadata.filter.ColumnTupleFilter; -import org.apache.kylin.metadata.filter.CompareTupleFilter; -import org.apache.kylin.metadata.filter.ConstantTupleFilter; -import org.apache.kylin.metadata.filter.LogicalTupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; -import org.apache.kylin.metadata.measure.LongMutable; -import org.apache.kylin.metadata.measure.serializer.StringSerializer; -import org.apache.kylin.metadata.model.DataType; -import org.apache.kylin.metadata.model.TblColRef; -import org.junit.Test; - -import com.google.common.collect.Lists; - -public class SimpleInvertedIndexTest { - - GTInfo info; - GTInvertedIndex index; - ArrayList<CompareTupleFilter> basicFilters = Lists.newArrayList(); - ArrayList<ConciseSet> basicResults = Lists.newArrayList(); - - public SimpleInvertedIndexTest() { - - info = UnitTestSupport.advancedInfo(); - TblColRef colA = info.colRef(0); - - // block i contains value "i", the last is NULL - index = new GTInvertedIndex(info); - GTRowBlock mockBlock = GTRowBlock.allocate(info); - GTRowBlock.Writer writer = mockBlock.getWriter(); - GTRecord record = new GTRecord(info); - for (int i = 0; i < 10; i++) { - record.setValues(i < 9 ? "" + i : null, "", "", new LongMutable(0), new BigDecimal(0)); - for (int j = 0; j < info.getRowBlockSize(); j++) { - writer.append(record); - } - writer.readyForFlush(); - index.add(mockBlock); - - writer.clearForNext(); - } - - basicFilters.add(compare(colA, FilterOperatorEnum.ISNULL)); - basicResults.add(set(9)); - - basicFilters.add(compare(colA, FilterOperatorEnum.ISNOTNULL)); - basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)); - - basicFilters.add(compare(colA, FilterOperatorEnum.EQ, 0)); - basicResults.add(set(0)); - - basicFilters.add(compare(colA, FilterOperatorEnum.NEQ, 0)); - basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)); - - basicFilters.add(compare(colA, FilterOperatorEnum.IN, 0, 5)); - basicResults.add(set(0, 5)); - - basicFilters.add(compare(colA, FilterOperatorEnum.NOTIN, 0, 5)); - basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)); - - basicFilters.add(compare(colA, FilterOperatorEnum.LT, 3)); - basicResults.add(set(0, 1, 2)); - - basicFilters.add(compare(colA, FilterOperatorEnum.LTE, 3)); - basicResults.add(set(0, 1, 2, 3)); - - basicFilters.add(compare(colA, FilterOperatorEnum.GT, 3)); - basicResults.add(set(4, 5, 6, 7, 8)); - - basicFilters.add(compare(colA, FilterOperatorEnum.GTE, 3)); - basicResults.add(set(3, 4, 5, 6, 7, 8)); - } - - @Test - public void testBasics() { - for (int i = 0; i < basicFilters.size(); i++) { - assertEquals(basicResults.get(i), index.filter(basicFilters.get(i))); - } - } - - @Test - public void testLogicalAnd() { - for (int i = 0; i < basicFilters.size(); i++) { - for (int j = 0; j < basicFilters.size(); j++) { - LogicalTupleFilter f = logical(FilterOperatorEnum.AND, basicFilters.get(i), basicFilters.get(j)); - ConciseSet r = basicResults.get(i).clone(); - r.retainAll(basicResults.get(j)); - assertEquals(r, index.filter(f)); - } - } - } - - @Test - public void testLogicalOr() { - for (int i = 0; i < basicFilters.size(); i++) { - for (int j = 0; j < basicFilters.size(); j++) { - LogicalTupleFilter f = logical(FilterOperatorEnum.OR, basicFilters.get(i), basicFilters.get(j)); - ConciseSet r = basicResults.get(i).clone(); - r.addAll(basicResults.get(j)); - assertEquals(r, index.filter(f)); - } - } - } - - @Test - public void testNotEvaluable() { - ConciseSet all = set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9); - - CompareTupleFilter notEvaluable = compare(info.colRef(1), FilterOperatorEnum.EQ, 0); - assertEquals(all, index.filter(notEvaluable)); - - LogicalTupleFilter or = logical(FilterOperatorEnum.OR, basicFilters.get(0), notEvaluable); - assertEquals(all, index.filter(or)); - - LogicalTupleFilter and = logical(FilterOperatorEnum.AND, basicFilters.get(0), notEvaluable); - assertEquals(basicResults.get(0), index.filter(and)); - } - - public static CompareTupleFilter compare(TblColRef col, TupleFilter.FilterOperatorEnum op, int... ids) { - CompareTupleFilter filter = new CompareTupleFilter(op); - filter.addChild(columnFilter(col)); - for (int i : ids) { - filter.addChild(constFilter(i)); - } - return filter; - } - - public static LogicalTupleFilter logical(TupleFilter.FilterOperatorEnum op, TupleFilter... filters) { - LogicalTupleFilter filter = new LogicalTupleFilter(op); - for (TupleFilter f : filters) - filter.addChild(f); - return filter; - } - - public static ColumnTupleFilter columnFilter(TblColRef col) { - return new ColumnTupleFilter(col); - } - - public static ConstantTupleFilter constFilter(int id) { - byte[] space = new byte[10]; - ByteBuffer buf = ByteBuffer.wrap(space); - StringSerializer stringSerializer = new StringSerializer(DataType.getInstance("string")); - stringSerializer.serialize("" + id, buf); - ByteArray data = new ByteArray(buf.array(), buf.arrayOffset(), buf.position()); - return new ConstantTupleFilter(data); - } - - public static ConciseSet set(int... ints) { - ConciseSet set = new ConciseSet(); - for (int i : ints) - set.add(i); - return set; - } - - -}
