KYLIN-2495 query exception when integer column encoded as date/time encoding
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c5f3b228 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c5f3b228 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c5f3b228 Branch: refs/heads/KYLIN-2360 Commit: c5f3b2286987d12c31310e9c9670d469606e9d4c Parents: 70dda40 Author: Hongbin Ma <mahong...@apache.org> Authored: Fri Mar 10 15:57:10 2017 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Fri Mar 10 17:17:02 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/util/DateFormat.java | 16 ++--- .../apache/kylin/cube/model/RowKeyColDesc.java | 7 +-- .../kylin/dimension/AbstractDateDimEnc.java | 25 ++++---- .../org/apache/kylin/dimension/DateDimEnc.java | 64 ++++++++++++++++---- .../org/apache/kylin/dimension/TimeDimEnc.java | 13 ++-- .../kylin/measure/topn/TopNMeasureType.java | 12 +++- .../apache/kylin/dimension/DateDimEncTest.java | 2 +- 7 files changed, 88 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java index 2412543..29858f1 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java @@ -48,6 +48,10 @@ public class DateFormat { } return r; } + + public static String formatToCompactDateStr(long millis) { + return formatToDateStr(millis, COMPACT_DATE_PATTERN); + } public static String formatToDateStr(long millis) { return formatToDateStr(millis, DEFAULT_DATE_PATTERN); @@ -88,18 +92,6 @@ public class DateFormat { } public static long stringToMillis(String str) { - return stringToMillis(str, null); - } - - public static long stringToMillis(String str, String dateFormat) { - try { - if (dateFormat != null) { - return getDateFormat(dateFormat).parse(str).getTime(); - } - } catch (ParseException e) { - // given format does not work, proceed to below - } - // try to be smart and guess the date format if (isAllDigits(str)) { if (str.length() == 8) http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java index 1df73ad..b6f0a27 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java @@ -86,10 +86,9 @@ public class RowKeyColDesc implements java.io.Serializable { encoding = encodingName = TimeDimEnc.ENCODING_NAME; } } - // if (DateDimEnc.ENCODING_NAME.equals(encodingName) && type.isDate() == false) - // throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply date encoding"); - // if (TimeDimEnc.ENCODING_NAME.equals(encodingName) && type.isTimeFamily() == false) - // throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply time encoding"); + + encodingArgs = DateDimEnc.replaceEncodingArgs(encoding, encodingArgs, encodingName, type); + if (encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && (type.isIntegerFamily() || type.isNumberFamily())) throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply fixed_length encoding"); } http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java index a54bcda..9c0d0da 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java @@ -18,10 +18,6 @@ package org.apache.kylin.dimension; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.DateFormat; -import org.apache.kylin.metadata.datatype.DataTypeSerializer; - import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; @@ -29,20 +25,23 @@ import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Arrays; +import org.apache.kylin.common.util.BytesUtil; +import org.apache.kylin.metadata.datatype.DataTypeSerializer; + public class AbstractDateDimEnc extends DimensionEncoding { private static final long serialVersionUID = 1L; - interface IMillisCodec extends Serializable { - long millisToCode(long millis); + interface IValueCodec extends Serializable { + long valueToCode(String value); - long codeToMillis(long code); + String codeToValue(long code); } // ============================================================================ private int fixedLen; - private IMillisCodec codec; + private IValueCodec codec; - protected AbstractDateDimEnc(int fixedLen, IMillisCodec codec) { + protected AbstractDateDimEnc(int fixedLen, IValueCodec codec) { this.fixedLen = fixedLen; this.codec = codec; } @@ -59,8 +58,7 @@ public class AbstractDateDimEnc extends DimensionEncoding { return; } - long millis = DateFormat.stringToMillis(value); - long code = codec.millisToCode(millis); + long code = codec.valueToCode(value); BytesUtil.writeLong(code, output, outputOffset, fixedLen); } @@ -74,8 +72,7 @@ public class AbstractDateDimEnc extends DimensionEncoding { if (code < 0) throw new IllegalArgumentException(); - long millis = codec.codeToMillis(code); - return String.valueOf(millis); + return codec.codeToValue(code); } @Override @@ -137,7 +134,7 @@ public class AbstractDateDimEnc extends DimensionEncoding { @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { this.fixedLen = in.readInt(); - this.codec = (IMillisCodec) in.readObject(); + this.codec = (IValueCodec) in.readObject(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java index 3b390d9..fee6233 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java @@ -20,10 +20,13 @@ package org.apache.kylin.dimension; import java.io.Serializable; +import org.apache.kylin.common.util.DateFormat; +import org.apache.kylin.metadata.datatype.DataType; + /** * This encoding is meant to be IDENTICAL to DateStrDictionary for 100% backward compatibility. */ -public class DateDimEnc extends AbstractDateDimEnc implements Serializable{ +public class DateDimEnc extends AbstractDateDimEnc implements Serializable { private static final long serialVersionUID = 1L; public static final int ID_9999_12_31 = 3652426; @@ -38,24 +41,47 @@ public class DateDimEnc extends AbstractDateDimEnc implements Serializable{ @Override public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) { - return new DateDimEnc(); + return new DateDimEnc(args); } }; - public DateDimEnc() { - super(3, new IMillisCodec() { - private static final long serialVersionUID = 1L; + private static class DateDimValueCodec implements IValueCodec { + + private static final long serialVersionUID = 1L; + private DataType datatype = null; - @Override - public long millisToCode(long millis) { - return getNumOfDaysSince0000FromMillis(millis); + public DateDimValueCodec(String[] args) { + if (args != null && args.length == 1) { + datatype = DataType.getType(args[0]); } + } + + @Override + public long valueToCode(String value) { + //if data type is integer, DateFormat.stringToMillis recognizes format like "20001010" + long millis = DateFormat.stringToMillis(value); + + return getNumOfDaysSince0000FromMillis(millis); + } - @Override - public long codeToMillis(long code) { - return getMillisFromNumOfDaysSince0000(code); + @Override + public String codeToValue(long code) { + long millisFromNumOfDaysSince0000 = getMillisFromNumOfDaysSince0000(code); + if (datatype != null && datatype.isIntegerFamily()) { + return DateFormat.formatToCompactDateStr(millisFromNumOfDaysSince0000); + } else { + return String.valueOf(millisFromNumOfDaysSince0000); } - }); + } + } + + //keep this for ser/der + public DateDimEnc() { + super(3, new DateDimValueCodec(null)); + } + + public DateDimEnc(String[] args) { + super(3, new DateDimValueCodec(args)); } public static long getNumOfDaysSince0000FromMillis(long millis) { @@ -69,4 +95,18 @@ public class DateDimEnc extends AbstractDateDimEnc implements Serializable{ return millis; } + public static String[] replaceEncodingArgs(String encoding, String[] encodingArgs, String encodingName, DataType type) { + // https://issues.apache.org/jira/browse/KYLIN-2495 + if (DateDimEnc.ENCODING_NAME.equals(encodingName)) { + if (type.isIntegerFamily()) { + if (encodingArgs.length != 0) { + throw new IllegalArgumentException("Date encoding should not specify arguments: " + encoding); + } + return new String[] { type.toString() }; + } + } + + return encodingArgs; + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java index 4bef2b8..dcc99dd 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java @@ -20,10 +20,12 @@ package org.apache.kylin.dimension; import java.io.Serializable; +import org.apache.kylin.common.util.DateFormat; + /** * This encoding is meant to be IDENTICAL to TimeStrDictionary for 100% backward compatibility. */ -public class TimeDimEnc extends AbstractDateDimEnc implements Serializable{ +public class TimeDimEnc extends AbstractDateDimEnc implements Serializable { private static final long serialVersionUID = 1L; public static final String ENCODING_NAME = "time"; @@ -41,17 +43,18 @@ public class TimeDimEnc extends AbstractDateDimEnc implements Serializable{ }; public TimeDimEnc() { - super(4, new IMillisCodec() { + super(4, new IValueCodec() { private static final long serialVersionUID = 1L; @Override - public long millisToCode(long millis) { + public long valueToCode(String value) { + long millis = DateFormat.stringToMillis(value); return millis / 1000; } @Override - public long codeToMillis(long code) { - return code * 1000; + public String codeToValue(long code) { + return String.valueOf(code * 1000); } }); } http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java index 8c8b5a6..b7252a0 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java @@ -24,10 +24,10 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dimension.DateDimEnc; import org.apache.kylin.dimension.DictionaryDimEnc; import org.apache.kylin.dimension.DimensionEncoding; import org.apache.kylin.dimension.DimensionEncodingFactory; @@ -47,6 +47,8 @@ import org.apache.kylin.metadata.tuple.TupleInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.Lists; + public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { private static final Logger logger = LoggerFactory.getLogger(TopNMeasureType.class); @@ -155,7 +157,6 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { return topNCounter; } - @Override public TopNCounter<ByteArray> reEncodeDictionary(TopNCounter<ByteArray> value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) { TopNCounter<ByteArray> topNCounter = value; @@ -424,7 +425,12 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> { } } Object[] encodingConf = DimensionEncoding.parseEncodingConf(encoding); - dimensionEncodings[i] = DimensionEncodingFactory.create((String) encodingConf[0], (String[]) encodingConf[1], encodingVersion); + String encodingName = (String) encodingConf[0]; + String[] encodingArgs = (String[]) encodingConf[1]; + + encodingArgs = DateDimEnc.replaceEncodingArgs(encoding, encodingArgs, encodingName, literalCols.get(i).getType()); + + dimensionEncodings[i] = DimensionEncodingFactory.create(encodingName, encodingArgs, encodingVersion); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java b/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java index 0183b5f..082f3ce 100644 --- a/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java @@ -38,7 +38,7 @@ public class DateDimEncTest { @Before public void setup() { - enc = new DateDimEnc(); + enc = new DateDimEnc(null); buf = new byte[enc.getLengthOfEncoding()]; }