Repository: kylin Updated Branches: refs/heads/master f0d2be3c9 -> 8babee8d0
deprecate only IntegerDimEnc and rename SlimLongDimEnc to IntegerDimEnc Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8babee8d Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8babee8d Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8babee8d Branch: refs/heads/master Commit: 8babee8d09f48d8ed60c70f16258402822ea81bf Parents: f0d2be3 Author: Yang Li <liy...@apache.org> Authored: Wed Sep 21 06:53:10 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Wed Sep 21 06:53:10 2016 +0800 ---------------------------------------------------------------------- .../gridtable/DimEncodingPreserveOrderTest.java | 6 +- .../dimension/DimensionEncodingFactory.java | 2 +- .../org/apache/kylin/dimension/IntDimEnc.java | 197 +++++++++++++++++ .../apache/kylin/dimension/IntegerDimEnc.java | 39 +++- .../apache/kylin/dimension/SlimLongDimEnc.java | 219 ------------------- .../apache/kylin/dimension/IntDimEncTest.java | 131 +++++++++++ .../kylin/dimension/IntegerDimEncTest.java | 53 +++-- .../kylin/dimension/SlimLongDimEncTest.java | 151 ------------- 8 files changed, 400 insertions(+), 398 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/8babee8d/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java index 0c84ed0..ffd43e5 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java @@ -27,7 +27,7 @@ import org.apache.kylin.common.util.Bytes; import org.apache.kylin.dimension.DimensionEncoding; import org.apache.kylin.dimension.FixedLenHexDimEnc; import org.apache.kylin.dimension.OneMoreByteVLongDimEnc; -import org.apache.kylin.dimension.SlimLongDimEnc; +import org.apache.kylin.dimension.IntegerDimEnc; import org.junit.Test; import com.google.common.collect.Lists; @@ -52,7 +52,7 @@ public class DimEncodingPreserveOrderTest { @Test public void testVLongDimEncPreserveOrder() { - SlimLongDimEnc enc = new SlimLongDimEnc(2); + IntegerDimEnc enc = new IntegerDimEnc(2); List<ByteArray> encodedValues = Lists.newArrayList(); encodedValues.add(encode(enc, -32767L)); encodedValues.add(encode(enc, -10000L)); @@ -68,7 +68,7 @@ public class DimEncodingPreserveOrderTest { @Test public void testVLongDimEncPreserveOrder2() { - SlimLongDimEnc enc = new SlimLongDimEnc(8); + IntegerDimEnc enc = new IntegerDimEnc(8); List<ByteArray> encodedValues = Lists.newArrayList(); encodedValues.add(encode(enc, -Long.MAX_VALUE)); encodedValues.add(encode(enc, -10000L)); http://git-wip-us.apache.org/repos/asf/kylin/blob/8babee8d/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java index 739fa25..27bebd7 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java @@ -74,8 +74,8 @@ public abstract class DimensionEncodingFactory { // built-in encodings, note dictionary is a special case map.put(FixedLenDimEnc.ENCODING_NAME, new FixedLenDimEnc.Factory()); + map.put(IntDimEnc.ENCODING_NAME, new IntDimEnc.Factory()); map.put(IntegerDimEnc.ENCODING_NAME, new IntegerDimEnc.Factory()); - map.put(SlimLongDimEnc.ENCODING_NAME, new SlimLongDimEnc.Factory()); map.put(FixedLenHexDimEnc.ENCODING_NAME, new FixedLenHexDimEnc.Factory()); map.put(DateDimEnc.ENCODING_NAME, new DateDimEnc.Factory()); map.put(TimeDimEnc.ENCODING_NAME, new TimeDimEnc.Factory()); http://git-wip-us.apache.org/repos/asf/kylin/blob/8babee8d/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java new file mode 100644 index 0000000..88af716 --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.dimension; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.nio.ByteBuffer; +import java.util.Arrays; + +import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.common.util.BytesUtil; +import org.apache.kylin.metadata.datatype.DataTypeSerializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * deprecated use IntegerDimEnc instead + * @deprecated + */ +public class IntDimEnc extends DimensionEncoding { + private static final long serialVersionUID = 1L; + + private static Logger logger = LoggerFactory.getLogger(IntDimEnc.class); + + private static final long[] CAP = { 0, 0xffL, 0xffffL, 0xffffffL, 0xffffffffL, 0xffffffffffL, 0xffffffffffffL, 0xffffffffffffffL, Long.MAX_VALUE }; + + public static final String ENCODING_NAME = "int"; + + public static class Factory extends DimensionEncodingFactory { + @Override + public String getSupportedEncodingName() { + return ENCODING_NAME; + } + + @Override + public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) { + return new IntDimEnc(Integer.parseInt(args[0])); + } + }; + + // ============================================================================ + + private int fixedLen; + + transient private int avoidVerbose = 0; + + //no-arg constructor is required for Externalizable + public IntDimEnc() { + } + + public IntDimEnc(int len) { + if (len <= 0 || len >= CAP.length) + throw new IllegalArgumentException(); + + this.fixedLen = len; + } + + @Override + public int getLengthOfEncoding() { + return fixedLen; + } + + @Override + public void encode(byte[] value, int valueLen, byte[] output, int outputOffset) { + if (value == null) { + Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL); + return; + } + + encode(Bytes.toString(value, 0, valueLen), output, outputOffset); + } + + void encode(String valueStr, byte[] output, int outputOffset) { + if (valueStr == null) { + Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL); + return; + } + + long integer = Long.parseLong(valueStr); + if (integer > CAP[fixedLen]) { + if (avoidVerbose++ % 10000 == 0) { + logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueStr + ", will truncate, hit times:" + avoidVerbose); + } + } + + BytesUtil.writeLong(integer, output, outputOffset, fixedLen); + } + + @Override + public String decode(byte[] bytes, int offset, int len) { + if (isNull(bytes, offset, len)) { + return null; + } + + long integer = BytesUtil.readLong(bytes, offset, len); + return String.valueOf(integer); + } + + @Override + public DataTypeSerializer<Object> asDataTypeSerializer() { + return new IntegerSerializer(); + } + + public class IntegerSerializer extends DataTypeSerializer<Object> { + // be thread-safe and avoid repeated obj creation + private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>(); + + private byte[] currentBuf() { + byte[] buf = current.get(); + if (buf == null) { + buf = new byte[fixedLen]; + current.set(buf); + } + return buf; + } + + @Override + public void serialize(Object value, ByteBuffer out) { + byte[] buf = currentBuf(); + String valueStr = value == null ? null : value.toString(); + encode(valueStr, buf, 0); + out.put(buf); + } + + @Override + public Object deserialize(ByteBuffer in) { + byte[] buf = currentBuf(); + in.get(buf); + return decode(buf, 0, buf.length); + } + + @Override + public int peekLength(ByteBuffer in) { + return fixedLen; + } + + @Override + public int maxLength() { + return fixedLen; + } + + @Override + public int getStorageBytesEstimate() { + return fixedLen; + } + + @Override + public Object valueOf(String str) { + return str; + } + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + out.writeShort(fixedLen); + } + + @Override + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + fixedLen = in.readShort(); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + IntDimEnc that = (IntDimEnc) o; + + return fixedLen == that.fixedLen; + + } + + @Override + public int hashCode() { + return fixedLen; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8babee8d/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java index b87e46a..a8cb510 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,16 +31,24 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * deprecated use SlimLongDimEnc instead + * replacement for IntegerDimEnc, the diff is VLongDimEnc supports negative values */ public class IntegerDimEnc extends DimensionEncoding { private static final long serialVersionUID = 1L; private static Logger logger = LoggerFactory.getLogger(IntegerDimEnc.class); - private static final long[] CAP = { 0, 0xffL, 0xffffL, 0xffffffL, 0xffffffffL, 0xffffffffffL, 0xffffffffffffL, 0xffffffffffffffL, Long.MAX_VALUE }; + private static final long[] CAP = { 0, 0x7fL, 0x7fffL, 0x7fffffL, 0x7fffffffL, 0x7fffffffffL, 0x7fffffffffffL, 0x7fffffffffffffL, 0x7fffffffffffffffL }; + private static final long[] MASK = { 0, 0xffL, 0xffffL, 0xffffffL, 0xffffffffL, 0xffffffffffL, 0xffffffffffffL, 0xffffffffffffffL, 0xffffffffffffffffL }; + private static final long[] TAIL = { 0, 0x80L, 0x8000L, 0x800000L, 0x80000000L, 0x8000000000L, 0x800000000000L, 0x80000000000000L, 0x8000000000000000L }; + static { + for (int i = 1; i < TAIL.length; ++i) { + long head = ~MASK[i]; + TAIL[i] = head | TAIL[i]; + } + } - public static final String ENCODING_NAME = "int"; + public static final String ENCODING_NAME = "integer"; public static class Factory extends DimensionEncodingFactory { @Override @@ -59,6 +67,7 @@ public class IntegerDimEnc extends DimensionEncoding { private int fixedLen; transient private int avoidVerbose = 0; + transient private int avoidVerbose2 = 0; //no-arg constructor is required for Externalizable public IntegerDimEnc() { @@ -93,13 +102,19 @@ public class IntegerDimEnc extends DimensionEncoding { } long integer = Long.parseLong(valueStr); - if (integer > CAP[fixedLen]) { + if (integer > CAP[fixedLen] || integer < TAIL[fixedLen]) { if (avoidVerbose++ % 10000 == 0) { logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueStr + ", will truncate, hit times:" + avoidVerbose); } } - BytesUtil.writeLong(integer, output, outputOffset, fixedLen); + if (integer == TAIL[fixedLen]) { + if (avoidVerbose2++ % 10000 == 0) { + logger.warn("Value " + valueStr + " does not fit into " + fixedLen + " bytes "); + } + } + + BytesUtil.writeLong(integer + CAP[fixedLen], output, outputOffset, fixedLen);//apply an offset to preserve binary order, overflow is okay } @Override @@ -108,7 +123,15 @@ public class IntegerDimEnc extends DimensionEncoding { return null; } - long integer = BytesUtil.readLong(bytes, offset, len); + long integer = BytesUtil.readLong(bytes, offset, len) - CAP[fixedLen]; + + //only take useful bytes + integer = integer & MASK[fixedLen]; + boolean positive = (integer & ((0x80) << ((fixedLen - 1) << 3))) == 0; + if (!positive) { + integer |= (~MASK[fixedLen]); + } + return String.valueOf(integer); } http://git-wip-us.apache.org/repos/asf/kylin/blob/8babee8d/core-metadata/src/main/java/org/apache/kylin/dimension/SlimLongDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/SlimLongDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/SlimLongDimEnc.java deleted file mode 100644 index 4ac871f..0000000 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/SlimLongDimEnc.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.dimension; - -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; -import java.nio.ByteBuffer; -import java.util.Arrays; - -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.metadata.datatype.DataTypeSerializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * replacement for IntegerDimEnc, the diff is VLongDimEnc supports negative values - */ -public class SlimLongDimEnc extends DimensionEncoding { - private static final long serialVersionUID = 1L; - - private static Logger logger = LoggerFactory.getLogger(SlimLongDimEnc.class); - - private static final long[] CAP = { 0, 0x7fL, 0x7fffL, 0x7fffffL, 0x7fffffffL, 0x7fffffffffL, 0x7fffffffffffL, 0x7fffffffffffffL, 0x7fffffffffffffffL }; - private static final long[] MASK = { 0, 0xffL, 0xffffL, 0xffffffL, 0xffffffffL, 0xffffffffffL, 0xffffffffffffL, 0xffffffffffffffL, 0xffffffffffffffffL }; - private static final long[] TAIL = { 0, 0x80L, 0x8000L, 0x800000L, 0x80000000L, 0x8000000000L, 0x800000000000L, 0x80000000000000L, 0x8000000000000000L }; - static { - for (int i = 1; i < TAIL.length; ++i) { - long head = ~MASK[i]; - TAIL[i] = head | TAIL[i]; - } - } - - public static final String ENCODING_NAME = "slimlong"; - - public static class Factory extends DimensionEncodingFactory { - @Override - public String getSupportedEncodingName() { - return ENCODING_NAME; - } - - @Override - public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) { - return new SlimLongDimEnc(Integer.parseInt(args[0])); - } - }; - - // ============================================================================ - - private int fixedLen; - - transient private int avoidVerbose = 0; - transient private int avoidVerbose2 = 0; - - //no-arg constructor is required for Externalizable - public SlimLongDimEnc() { - } - - public SlimLongDimEnc(int len) { - if (len <= 0 || len >= CAP.length) - throw new IllegalArgumentException(); - - this.fixedLen = len; - } - - @Override - public int getLengthOfEncoding() { - return fixedLen; - } - - @Override - public void encode(byte[] value, int valueLen, byte[] output, int outputOffset) { - if (value == null) { - Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL); - return; - } - - encode(Bytes.toString(value, 0, valueLen), output, outputOffset); - } - - void encode(String valueStr, byte[] output, int outputOffset) { - if (valueStr == null) { - Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL); - return; - } - - long integer = Long.parseLong(valueStr); - if (integer > CAP[fixedLen] || integer < TAIL[fixedLen]) { - if (avoidVerbose++ % 10000 == 0) { - logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueStr + ", will truncate, hit times:" + avoidVerbose); - } - } - - if (integer == TAIL[fixedLen]) { - if (avoidVerbose2++ % 10000 == 0) { - logger.warn("Value " + valueStr + " does not fit into " + fixedLen + " bytes "); - } - } - - BytesUtil.writeLong(integer + CAP[fixedLen], output, outputOffset, fixedLen);//apply an offset to preserve binary order, overflow is okay - } - - @Override - public String decode(byte[] bytes, int offset, int len) { - if (isNull(bytes, offset, len)) { - return null; - } - - long integer = BytesUtil.readLong(bytes, offset, len) - CAP[fixedLen]; - - //only take useful bytes - integer = integer & MASK[fixedLen]; - boolean positive = (integer & ((0x80) << ((fixedLen - 1) << 3))) == 0; - if (!positive) { - integer |= (~MASK[fixedLen]); - } - - return String.valueOf(integer); - } - - @Override - public DataTypeSerializer<Object> asDataTypeSerializer() { - return new IntegerSerializer(); - } - - public class IntegerSerializer extends DataTypeSerializer<Object> { - // be thread-safe and avoid repeated obj creation - private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>(); - - private byte[] currentBuf() { - byte[] buf = current.get(); - if (buf == null) { - buf = new byte[fixedLen]; - current.set(buf); - } - return buf; - } - - @Override - public void serialize(Object value, ByteBuffer out) { - byte[] buf = currentBuf(); - String valueStr = value == null ? null : value.toString(); - encode(valueStr, buf, 0); - out.put(buf); - } - - @Override - public Object deserialize(ByteBuffer in) { - byte[] buf = currentBuf(); - in.get(buf); - return decode(buf, 0, buf.length); - } - - @Override - public int peekLength(ByteBuffer in) { - return fixedLen; - } - - @Override - public int maxLength() { - return fixedLen; - } - - @Override - public int getStorageBytesEstimate() { - return fixedLen; - } - - @Override - public Object valueOf(String str) { - return str; - } - } - - @Override - public void writeExternal(ObjectOutput out) throws IOException { - out.writeShort(fixedLen); - } - - @Override - public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { - fixedLen = in.readShort(); - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - SlimLongDimEnc that = (SlimLongDimEnc) o; - - return fixedLen == that.fixedLen; - - } - - @Override - public int hashCode() { - return fixedLen; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/8babee8d/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java b/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java new file mode 100644 index 0000000..280a242 --- /dev/null +++ b/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.dimension; + +import java.nio.ByteBuffer; + +import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.metadata.datatype.DataTypeSerializer; +import org.junit.Assert; +import org.junit.Test; + +/** + * Deprecated. use VLongDimEnc instead + * @deprecated + */ +public class IntDimEncTest { + + @Test + public void testConstructor() { + try { + new IntDimEnc(0); + Assert.fail(); + } catch (IllegalArgumentException e) { + // expect + } + try { + new IntDimEnc(9); + Assert.fail(); + } catch (IllegalArgumentException e) { + // expect + } + new IntDimEnc(8); + } + + @Test + public void testNull() { + for (int i = 1; i < 9; i++) { + IntDimEnc enc = new IntDimEnc(i); + + byte[] buf = new byte[enc.getLengthOfEncoding()]; + enc.encode(null, 0, buf, 0); + Assert.assertTrue(DimensionEncoding.isNull(buf, 0, buf.length)); + String decode = enc.decode(buf, 0, buf.length); + Assert.assertEquals(null, decode); + + buf = new byte[enc.getLengthOfEncoding()]; + DataTypeSerializer<Object> ser = enc.asDataTypeSerializer(); + ser.serialize(null, ByteBuffer.wrap(buf)); + Assert.assertTrue(DimensionEncoding.isNull(buf, 0, buf.length)); + decode = (String) ser.deserialize(ByteBuffer.wrap(buf)); + Assert.assertEquals(null, decode); + } + } + + @Test + public void testEncodeDecode() { + IntDimEnc enc = new IntDimEnc(2); + testEncodeDecode(enc, 0); + testEncodeDecode(enc, 100); + testEncodeDecode(enc, 10000); + testEncodeDecode(enc, 65534); + try { + testEncodeDecode(enc, 65535); + Assert.fail(); + } catch (Throwable e) { + Assert.assertEquals("expected:<65535> but was:<null>", e.getMessage()); + } + try { + testEncodeDecode(enc, 65536); + Assert.fail(); + } catch (Throwable e) { + Assert.assertEquals("expected:<[65536]> but was:<[0]>", e.getMessage()); + } + } + + private void testEncodeDecode(IntDimEnc enc, long value) { + byte[] buf = new byte[enc.getLengthOfEncoding()]; + String valueStr = "" + value; + byte[] bytes = Bytes.toBytes(valueStr); + enc.encode(bytes, bytes.length, buf, 0); + String decode = enc.decode(buf, 0, buf.length); + Assert.assertEquals(valueStr, decode); + } + + @Test + public void testSerDes() { + IntDimEnc enc = new IntDimEnc(2); + testSerDes(enc, 0); + testSerDes(enc, 100); + testSerDes(enc, 10000); + testSerDes(enc, 65534); + try { + testSerDes(enc, 65535); + Assert.fail(); + } catch (Throwable e) { + Assert.assertEquals("expected:<65535> but was:<null>", e.getMessage()); + } + try { + testSerDes(enc, 65536); + Assert.fail(); + } catch (Throwable e) { + Assert.assertEquals("expected:<[65536]> but was:<[0]>", e.getMessage()); + } + } + + private void testSerDes(IntDimEnc enc, long value) { + DataTypeSerializer<Object> ser = enc.asDataTypeSerializer(); + byte[] buf = new byte[enc.getLengthOfEncoding()]; + String valueStr = "" + value; + ser.serialize(valueStr, ByteBuffer.wrap(buf)); + String decode = (String) ser.deserialize(ByteBuffer.wrap(buf)); + Assert.assertEquals(valueStr, decode); + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8babee8d/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java b/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java index fccd8b9..1bdb1d0 100644 --- a/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,9 +25,6 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer; import org.junit.Assert; import org.junit.Test; -/** - * Deprecated. use VLongDimEnc instead - */ public class IntegerDimEncTest { @Test @@ -73,24 +70,45 @@ public class IntegerDimEncTest { testEncodeDecode(enc, 0); testEncodeDecode(enc, 100); testEncodeDecode(enc, 10000); - testEncodeDecode(enc, 65534); + testEncodeDecode(enc, 32767); + testEncodeDecode(enc, -100); + testEncodeDecode(enc, -10000); + testEncodeDecode(enc, -32767); try { - testEncodeDecode(enc, 65535); + testEncodeDecode(enc, 32768); Assert.fail(); } catch (Throwable e) { - Assert.assertEquals("expected:<65535> but was:<null>", e.getMessage()); + Assert.assertEquals("expected:<32768> but was:<null>", e.getMessage()); } try { - testEncodeDecode(enc, 65536); + testEncodeDecode(enc, -32768); Assert.fail(); } catch (Throwable e) { - Assert.assertEquals("expected:<[65536]> but was:<[0]>", e.getMessage()); + Assert.assertEquals("expected:<-32768> but was:<null>", e.getMessage()); + } + } + + @Test + public void testEncodeDecode2() { + IntegerDimEnc enc = new IntegerDimEnc(8); + testEncodeDecode(enc, 0); + testEncodeDecode(enc, 100); + testEncodeDecode(enc, 10000); + testEncodeDecode(enc, Long.MAX_VALUE); + testEncodeDecode(enc, -100); + testEncodeDecode(enc, -10000); + testEncodeDecode(enc, -Long.MAX_VALUE); + try { + testEncodeDecode(enc, Long.MIN_VALUE); + Assert.fail(); + } catch (Throwable e) { + Assert.assertEquals("expected:<-9223372036854775808> but was:<null>", e.getMessage()); } } private void testEncodeDecode(IntegerDimEnc enc, long value) { - byte[] buf = new byte[enc.getLengthOfEncoding()]; String valueStr = "" + value; + byte[] buf = new byte[enc.getLengthOfEncoding()]; byte[] bytes = Bytes.toBytes(valueStr); enc.encode(bytes, bytes.length, buf, 0); String decode = enc.decode(buf, 0, buf.length); @@ -103,18 +121,21 @@ public class IntegerDimEncTest { testSerDes(enc, 0); testSerDes(enc, 100); testSerDes(enc, 10000); - testSerDes(enc, 65534); + testSerDes(enc, 32767); + testSerDes(enc, -100); + testSerDes(enc, -10000); + testSerDes(enc, -32767); try { - testSerDes(enc, 65535); + testSerDes(enc, 32768); Assert.fail(); } catch (Throwable e) { - Assert.assertEquals("expected:<65535> but was:<null>", e.getMessage()); + Assert.assertEquals("expected:<32768> but was:<null>", e.getMessage()); } try { - testSerDes(enc, 65536); + testSerDes(enc, -32768); Assert.fail(); } catch (Throwable e) { - Assert.assertEquals("expected:<[65536]> but was:<[0]>", e.getMessage()); + Assert.assertEquals("expected:<-32768> but was:<null>", e.getMessage()); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/8babee8d/core-metadata/src/test/java/org/apache/kylin/dimension/SlimLongDimEncTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/dimension/SlimLongDimEncTest.java b/core-metadata/src/test/java/org/apache/kylin/dimension/SlimLongDimEncTest.java deleted file mode 100644 index a1e9516..0000000 --- a/core-metadata/src/test/java/org/apache/kylin/dimension/SlimLongDimEncTest.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.dimension; - -import java.nio.ByteBuffer; - -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.metadata.datatype.DataTypeSerializer; -import org.junit.Assert; -import org.junit.Test; - -public class SlimLongDimEncTest { - - @Test - public void testConstructor() { - try { - new SlimLongDimEnc(0); - Assert.fail(); - } catch (IllegalArgumentException e) { - // expect - } - try { - new SlimLongDimEnc(9); - Assert.fail(); - } catch (IllegalArgumentException e) { - // expect - } - new SlimLongDimEnc(8); - } - - @Test - public void testNull() { - for (int i = 1; i < 9; i++) { - SlimLongDimEnc enc = new SlimLongDimEnc(i); - - byte[] buf = new byte[enc.getLengthOfEncoding()]; - enc.encode(null, 0, buf, 0); - Assert.assertTrue(DimensionEncoding.isNull(buf, 0, buf.length)); - String decode = enc.decode(buf, 0, buf.length); - Assert.assertEquals(null, decode); - - buf = new byte[enc.getLengthOfEncoding()]; - DataTypeSerializer<Object> ser = enc.asDataTypeSerializer(); - ser.serialize(null, ByteBuffer.wrap(buf)); - Assert.assertTrue(DimensionEncoding.isNull(buf, 0, buf.length)); - decode = (String) ser.deserialize(ByteBuffer.wrap(buf)); - Assert.assertEquals(null, decode); - } - } - - @Test - public void testEncodeDecode() { - SlimLongDimEnc enc = new SlimLongDimEnc(2); - testEncodeDecode(enc, 0); - testEncodeDecode(enc, 100); - testEncodeDecode(enc, 10000); - testEncodeDecode(enc, 32767); - testEncodeDecode(enc, -100); - testEncodeDecode(enc, -10000); - testEncodeDecode(enc, -32767); - try { - testEncodeDecode(enc, 32768); - Assert.fail(); - } catch (Throwable e) { - Assert.assertEquals("expected:<32768> but was:<null>", e.getMessage()); - } - try { - testEncodeDecode(enc, -32768); - Assert.fail(); - } catch (Throwable e) { - Assert.assertEquals("expected:<-32768> but was:<null>", e.getMessage()); - } - } - - @Test - public void testEncodeDecode2() { - SlimLongDimEnc enc = new SlimLongDimEnc(8); - testEncodeDecode(enc, 0); - testEncodeDecode(enc, 100); - testEncodeDecode(enc, 10000); - testEncodeDecode(enc, Long.MAX_VALUE); - testEncodeDecode(enc, -100); - testEncodeDecode(enc, -10000); - testEncodeDecode(enc, -Long.MAX_VALUE); - try { - testEncodeDecode(enc, Long.MIN_VALUE); - Assert.fail(); - } catch (Throwable e) { - Assert.assertEquals("expected:<-9223372036854775808> but was:<null>", e.getMessage()); - } - } - - private void testEncodeDecode(SlimLongDimEnc enc, long value) { - String valueStr = "" + value; - byte[] buf = new byte[enc.getLengthOfEncoding()]; - byte[] bytes = Bytes.toBytes(valueStr); - enc.encode(bytes, bytes.length, buf, 0); - String decode = enc.decode(buf, 0, buf.length); - Assert.assertEquals(valueStr, decode); - } - - @Test - public void testSerDes() { - SlimLongDimEnc enc = new SlimLongDimEnc(2); - testSerDes(enc, 0); - testSerDes(enc, 100); - testSerDes(enc, 10000); - testSerDes(enc, 32767); - testSerDes(enc, -100); - testSerDes(enc, -10000); - testSerDes(enc, -32767); - try { - testSerDes(enc, 32768); - Assert.fail(); - } catch (Throwable e) { - Assert.assertEquals("expected:<32768> but was:<null>", e.getMessage()); - } - try { - testSerDes(enc, -32768); - Assert.fail(); - } catch (Throwable e) { - Assert.assertEquals("expected:<-32768> but was:<null>", e.getMessage()); - } - } - - private void testSerDes(SlimLongDimEnc enc, long value) { - DataTypeSerializer<Object> ser = enc.asDataTypeSerializer(); - byte[] buf = new byte[enc.getLengthOfEncoding()]; - String valueStr = "" + value; - ser.serialize(valueStr, ByteBuffer.wrap(buf)); - String decode = (String) ser.deserialize(ByteBuffer.wrap(buf)); - Assert.assertEquals(valueStr, decode); - } - -}
