Repository: kylin Updated Branches: refs/heads/KYLIN-1122 1e44c595b -> 8814a432d
KYLIN-1122 add UT and bug fixed Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fd1898e5 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fd1898e5 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fd1898e5 Branch: refs/heads/KYLIN-1122 Commit: fd1898e58c918b0f8bd2698407a910ae8504b3bf Parents: ab7a76d Author: Xiaoyu Wang <[email protected]> Authored: Tue Jan 12 18:53:30 2016 +0800 Committer: Xiaoyu Wang <[email protected]> Committed: Tue Jan 12 18:53:30 2016 +0800 ---------------------------------------------------------------------- .../kylin/measure/basic/BasicMeasureType.java | 4 -- .../kylin/measure/raw/RawMeasureType.java | 2 +- .../apache/kylin/measure/raw/RawSerializer.java | 29 +++++---- .../kylin/metadata/model/FunctionDesc.java | 1 - .../kylin/measure/raw/RawAggregatorTest.java | 35 +++++++++++ .../kylin/measure/raw/RawSerializerTest.java | 64 ++++++++++++++++++++ 6 files changed, 118 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/fd1898e5/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java index 9447ab0..b8d201e 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java @@ -77,10 +77,6 @@ public class BasicMeasureType extends MeasureType { if (rtype.isNumberFamily() == false) { throw new IllegalArgumentException("Return type for function " + funcName + " must be one of " + DataType.NUMBER_FAMILY); } - } else if (funcName.equals(FunctionDesc.FUNC_RAW)) { - if (!FunctionDesc.FUNC_RAW.equalsIgnoreCase(rtype.getName())) { - throw new IllegalArgumentException("Return type for function " + funcName + " must be raw "); - } } else { KylinConfig config = KylinConfig.getInstanceFromEnv(); if (config.isQueryIgnoreUnknownFunction() == false) http://git-wip-us.apache.org/repos/asf/kylin/blob/fd1898e5/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java index 92044de..755d7f6 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java @@ -214,7 +214,7 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> { ByteArray raw = rawIterator.next(); int key = BytesUtil.readUnsigned(raw.array(), 0, raw.array().length); String colValue = rawColDict.getValueFromId(key); - tuple.setMeasureValue(literalTupleIdx, colValue); + tuple.setDimensionValue(literalTupleIdx, colValue); } }; } http://git-wip-us.apache.org/repos/asf/kylin/blob/fd1898e5/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java index 884f777..25c70bc 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java @@ -30,6 +30,9 @@ import java.util.List; @SuppressWarnings("unused") public class RawSerializer extends DataTypeSerializer<List<ByteArray>> { + //one dictionary id value need 2 bytes, this buffer can contain ~ 512 * 1024 values + //FIXME to config this and RowConstants.ROWVALUE_BUFFER_SIZE in properties file + public static final int RAW_BUFFER_SIZE = 1024 * 1024;//1M public RawSerializer(DataType dataType) { } @@ -38,9 +41,9 @@ public class RawSerializer extends DataTypeSerializer<List<ByteArray>> { public int peekLength(ByteBuffer in) { int mark = in.position(); int len = 0; - if(in.hasRemaining()) { - int size = in.getInt(); - int bytes = in.getInt(); + if (in.hasRemaining()) { + int size = BytesUtil.readVInt(in); + int bytes = BytesUtil.readVInt(in); len = in.position() - mark + bytes; } in.position(mark); @@ -49,26 +52,30 @@ public class RawSerializer extends DataTypeSerializer<List<ByteArray>> { @Override public int maxLength() { - return 1024 * 1024; + return RAW_BUFFER_SIZE; } @Override public int getStorageBytesEstimate() { - return 1024 * 1024; + return RAW_BUFFER_SIZE; } @Override public void serialize(List<ByteArray> value, ByteBuffer out) { - if(value != null) { + if (value != null) { int bytes = 0; for (ByteArray array : value) { + //in BytesUtil.writeByteArray, writeVInt only need 1 byte, + //because dictionary id length is less than 127 + //so one value will need 2 bytes bytes += (array.length() + 1); } + //size and bytes value will need <=8 bytes + BytesUtil.writeVInt(value.size(), out); + BytesUtil.writeVInt(bytes, out); if (bytes > out.remaining()) { throw new RuntimeException("BufferOverflow! Please use one higher cardinality column for dimension column when build RAW cube!"); } - out.putInt(value.size()); - out.putInt(bytes); for (ByteArray array : value) { BytesUtil.writeByteArray(array.array(), out); } @@ -78,9 +85,9 @@ public class RawSerializer extends DataTypeSerializer<List<ByteArray>> { @Override public List<ByteArray> deserialize(ByteBuffer in) { List<ByteArray> value = null; - if(in.hasRemaining()) { - int size = in.getInt(); - int bytes = in.getInt(); + if (in.hasRemaining()) { + int size = BytesUtil.readVInt(in); + int bytes = BytesUtil.readVInt(in); value = new ArrayList<ByteArray>(size); for (int i = 0; i < size; i++) { value.add(new ByteArray(BytesUtil.readByteArray(in))); http://git-wip-us.apache.org/repos/asf/kylin/blob/fd1898e5/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java index 624a73c..36c8722 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java @@ -40,7 +40,6 @@ public class FunctionDesc { public static final String FUNC_MIN = "MIN"; public static final String FUNC_MAX = "MAX"; public static final String FUNC_COUNT = "COUNT"; - public static final String FUNC_RAW = "RAW"; public static final String PARAMETER_TYPE_CONSTANT = "constant"; public static final String PARAMETER_TYPE_COLUMN = "column"; http://git-wip-us.apache.org/repos/asf/kylin/blob/fd1898e5/core-metadata/src/test/java/org/apache/kylin/measure/raw/RawAggregatorTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/raw/RawAggregatorTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/raw/RawAggregatorTest.java new file mode 100644 index 0000000..da54664 --- /dev/null +++ b/core-metadata/src/test/java/org/apache/kylin/measure/raw/RawAggregatorTest.java @@ -0,0 +1,35 @@ +package org.apache.kylin.measure.raw; + + +import org.apache.kylin.common.util.ByteArray; +import org.apache.kylin.common.util.BytesUtil; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class RawAggregatorTest { + private RawAggregator agg = new RawAggregator(); + + @Test + public void testNormal(){ + int size = 100; + List<ByteArray> valueList = new ArrayList<ByteArray>(size); + for (Integer i = 0; i < size; i++) { + ByteArray key = new ByteArray(1); + BytesUtil.writeUnsigned(i, key.array(), 0, key.length()); + valueList.add(key); + } + agg.aggregate(valueList); + agg.aggregate(valueList); + assertEquals(valueList.size() * 2, agg.getState().size()); + } + + @Test + public void testNull(){ + agg.aggregate(null); + assertEquals(agg.getState().size(), 0); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/fd1898e5/core-metadata/src/test/java/org/apache/kylin/measure/raw/RawSerializerTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/raw/RawSerializerTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/raw/RawSerializerTest.java new file mode 100644 index 0000000..a870559 --- /dev/null +++ b/core-metadata/src/test/java/org/apache/kylin/measure/raw/RawSerializerTest.java @@ -0,0 +1,64 @@ +package org.apache.kylin.measure.raw; + +import org.apache.kylin.common.util.ByteArray; +import org.apache.kylin.common.util.BytesUtil; +import org.apache.kylin.metadata.datatype.DataType; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class RawSerializerTest { + static { + DataType.register("raw"); + } + + private RawSerializer rawSerializer = new RawSerializer(DataType.getType("raw")); + + @Test + public void testNormal() { + List<ByteArray> input = getValueList(1024); + List<ByteArray> output = doSAndD(input); + assertEquals(input, output); + } + + @Test + public void testNull() { + List<ByteArray> output = doSAndD(null); + assertEquals(output.size(), 0); + List<ByteArray> input = new ArrayList<ByteArray>(); + output = doSAndD(input); + assertEquals(input, output); + } + + @Test(expected = RuntimeException.class) + public void testOverflow() { + List<ByteArray> input = getValueList(512 * 1024); + doSAndD(input); + } + + private List<ByteArray> doSAndD(List<ByteArray> input) { + ByteBuffer out = ByteBuffer.allocate(rawSerializer.maxLength()); + out.mark(); + rawSerializer.serialize(input, out); + out.reset(); + return rawSerializer.deserialize(out); + } + + private List<ByteArray> getValueList(int size) { + if (size == -1) { + return null; + } + List<ByteArray> valueList = new ArrayList<ByteArray>(size); + for (Integer i = 0; i < size; i++) { + ByteArray key = new ByteArray(1); + BytesUtil.writeUnsigned(i, key.array(), 0, key.length()); + valueList.add(key); + } + return valueList; + } + +}
