Repository: kylin Updated Branches: refs/heads/master a82662bcd -> 11e20adbd
KYLIN-1766 Alter LookupTable to hold millis instead of yyyymmdd Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8a96b57f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8a96b57f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8a96b57f Branch: refs/heads/master Commit: 8a96b57f066ed58d55339d3b8e18bbf51c45ec34 Parents: a82662b Author: Yang Li <[email protected]> Authored: Sat Jun 18 22:34:04 2016 +0800 Committer: Yang Li <[email protected]> Committed: Sun Jun 19 14:15:51 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/cube/CubeManager.java | 2 - .../java/org/apache/kylin/cube/CubeSegment.java | 3 +- .../apache/kylin/cube/kv/RowKeyColumnOrder.java | 2 +- .../apache/kylin/cube/model/RowKeyColDesc.java | 4 +- .../kylin/gridtable/GTScanRangePlanner.java | 1 - .../kylin/dict/lookup/LookupBytesTable.java | 56 --------------- .../kylin/dict/lookup/LookupStringTable.java | 71 +++++++++++++++---- .../apache/kylin/dict/lookup/LookupTable.java | 18 +++-- .../kylin/dict/lookup/LookupTableTest.java | 19 +++-- .../metadata/filter/ConstantTupleFilter.java | 5 +- .../apache/kylin/engine/mr/LookupTableTest.java | 74 -------------------- .../resources/query/sql_derived/query11.sql | 26 +++++++ .../kylin/query/enumerator/OLAPEnumerator.java | 25 ++----- .../kylin/query/relnode/OLAPFilterRel.java | 25 ++++++- storage-hbase/pom.xml | 4 +- .../storage/hbase/cube/v1/CubeStorageQuery.java | 6 +- 16 files changed, 157 insertions(+), 184 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index a33cf00..caa2765 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -201,8 +201,6 @@ public class CubeManager implements IRealizationProvider { DictionaryInfo info = null; try { DictionaryManager dictMgr = getDictionaryManager(); - // logger.info("Using metadata url " + metadataUrl + - // " for DictionaryManager"); String dictResPath = cubeSeg.getDictResPath(col); if (dictResPath == null) return null; http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java index f79e06d..b9195fd 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java @@ -305,7 +305,8 @@ public class CubeSegment implements Comparable<CubeSegment>, IRealizationSegment public Dictionary<String> getDictionary(TblColRef col) { TblColRef reuseCol = getCubeDesc().getDictionaryReuseColumn(col); - return CubeManager.getInstance(this.getCubeInstance().getConfig()).getDictionary(this, reuseCol); + CubeManager cubeMgr = CubeManager.getInstance(this.getCubeInstance().getConfig()); + return cubeMgr.getDictionary(this, reuseCol); } public CubeDimEncMap getDimensionEncodingMap() { http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java index fea3736..23c8f6a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java @@ -32,7 +32,7 @@ abstract public class RowKeyColumnOrder implements Comparator<String> { public static final StringOrder STRING_ORDER = new StringOrder(); public static RowKeyColumnOrder getInstance(DataType type) { - if (type.isNumberFamily()) + if (type.isNumberFamily() || type.isDateTimeFamily()) return NUMBER_ORDER; else return STRING_ORDER; http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java index c4174a6..d4cfcb7 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java @@ -79,10 +79,10 @@ public class RowKeyColDesc { if (DictionaryDimEnc.ENCODING_NAME.equals(encodingName)) { DataType type = colRef.getType(); if (type.isDate()) { - encodingName = DateDimEnc.ENCODING_NAME; + encoding = encodingName = DateDimEnc.ENCODING_NAME; } if (type.isTime() || type.isTimestamp() || type.isDatetime()) { - encodingName = TimeDimEnc.ENCODING_NAME; + encoding = encodingName = TimeDimEnc.ENCODING_NAME; } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java index baf3428..7d31bf7 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java @@ -156,7 +156,6 @@ public class GTScanRangePlanner { //start key GTRecord compare to stop key GTRecord this.rangeStartEndComparator = getRangeStartEndComparator(comp); - this.gtFilter = gtFilter; this.gtStartAndEnd = gtStartAndEnd; this.gtPartitionCol = gtPartitionCol; http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java deleted file mode 100644 index 0758edc..0000000 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.dict.lookup; - -import java.io.IOException; - -import org.apache.kylin.common.util.ByteArray; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.metadata.model.TableDesc; -import org.apache.kylin.source.ReadableTable; - -/** - * @author yangli9 - * - */ -public class LookupBytesTable extends LookupTable<ByteArray> { - - public LookupBytesTable(TableDesc tableDesc, String[] keyColumns, ReadableTable table) throws IOException { - super(tableDesc, keyColumns, table); - } - - @Override - protected ByteArray[] convertRow(String[] cols) { - ByteArray[] r = new ByteArray[cols.length]; - for (int i = 0; i < cols.length; i++) { - r[i] = cols[i] == null ? null : new ByteArray(Bytes.toBytes(cols[i])); - } - return r; - } - - @Override - protected String toString(ByteArray cell) { - return cell.toString(); - } - - public Class<?> getType() { - return ByteArray.class; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java index 9f6346a..fd88fb0 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java @@ -19,8 +19,10 @@ package org.apache.kylin.dict.lookup; import java.io.IOException; +import java.util.Comparator; import org.apache.kylin.common.util.DateFormat; +import org.apache.kylin.metadata.datatype.DataType; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.source.ReadableTable; @@ -30,37 +32,80 @@ import org.apache.kylin.source.ReadableTable; * */ public class LookupStringTable extends LookupTable<String> { + + private static final Comparator<String> dateStrComparator = new Comparator<String>() { + @Override + public int compare(String o1, String o2) { + long l1 = Long.parseLong(o1); + long l2 = Long.parseLong(o2); + return Long.compare(l1, l2); + } + }; - int[] keyIndexOfDates; + private static final Comparator<String> numStrComparator = new Comparator<String>() { + @Override + public int compare(String o1, String o2) { + double d1 = Double.parseDouble(o1); + double d2 = Double.parseDouble(o2); + return Double.compare(d1, d2); + } + }; + + private static final Comparator<String> defaultStrComparator = new Comparator<String>() { + @Override + public int compare(String o1, String o2) { + return o1.compareTo(o2); + } + }; + + boolean[] colIsDateTime; + boolean[] colIsNumber; public LookupStringTable(TableDesc tableDesc, String[] keyColumns, ReadableTable table) throws IOException { super(tableDesc, keyColumns, table); } + + @Override + protected void init() throws IOException { + ColumnDesc[] cols = tableDesc.getColumns(); + colIsDateTime = new boolean[cols.length]; + colIsNumber = new boolean[cols.length]; + for (int i = 0; i < cols.length; i++) { + DataType t = cols[i].getType(); + colIsDateTime[i] = t.isDateTimeFamily(); + colIsNumber[i] = t.isNumberFamily(); + } + + super.init(); + } @Override protected String[] convertRow(String[] cols) { - if (keyIndexOfDates == null) { - keyIndexOfDates = new int[keyColumns.length]; - for (int i = 0; i < keyColumns.length; i++) { - ColumnDesc col = tableDesc.findColumnByName(keyColumns[i]); - keyIndexOfDates[i] = col.getType().isDateTimeFamily() ? col.getZeroBasedIndex() : -1; + for (int i = 0; i < cols.length; i++) { + if (colIsDateTime[i]) { + cols[i] = String.valueOf(DateFormat.stringToMillis(cols[i])); } } - - for (int i = 0; i < keyIndexOfDates.length; i++) { - int c = keyIndexOfDates[i]; - if (c >= 0) - cols[c] = String.valueOf(DateFormat.stringToMillis(cols[c])); - } return cols; } @Override + protected Comparator<String> getComparator(int idx) { + if (colIsDateTime[idx]) + return dateStrComparator; + else if (colIsNumber[idx]) + return numStrComparator; + else + return defaultStrComparator; + } + + @Override protected String toString(String cell) { return cell; } public Class<?> getType() { return String.class; - } + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java index 9abd28c..5a96df6 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java @@ -21,6 +21,7 @@ package org.apache.kylin.dict.lookup; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Comparator; import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -39,7 +40,7 @@ import com.google.common.collect.Sets; * * @author yangli9 */ -abstract public class LookupTable<T extends Comparable<T>> { +abstract public class LookupTable<T> { protected TableDesc tableDesc; protected String[] keyColumns; @@ -109,15 +110,18 @@ abstract public class LookupTable<T extends Comparable<T>> { public Pair<T, T> mapRange(String col, T beginValue, T endValue, String returnCol) { int colIdx = tableDesc.findColumnByName(col).getZeroBasedIndex(); int returnIdx = tableDesc.findColumnByName(returnCol).getZeroBasedIndex(); + Comparator<T> colComp = getComparator(colIdx); + Comparator<T> returnComp = getComparator(returnIdx); + T returnBegin = null; T returnEnd = null; for (T[] row : data.values()) { - if (between(beginValue, row[colIdx], endValue)) { + if (between(beginValue, row[colIdx], endValue, colComp)) { T returnValue = row[returnIdx]; - if (returnBegin == null || returnValue.compareTo(returnBegin) < 0) { + if (returnBegin == null || returnComp.compare(returnValue, returnBegin) < 0) { returnBegin = returnValue; } - if (returnEnd == null || returnValue.compareTo(returnEnd) > 0) { + if (returnEnd == null || returnComp.compare(returnValue, returnEnd) > 0) { returnEnd = returnValue; } } @@ -140,10 +144,12 @@ abstract public class LookupTable<T extends Comparable<T>> { return result; } - private boolean between(T beginValue, T v, T endValue) { - return (beginValue == null || beginValue.compareTo(v) <= 0) && (endValue == null || v.compareTo(endValue) <= 0); + private boolean between(T beginValue, T v, T endValue, Comparator<T> comp) { + return (beginValue == null || comp.compare(beginValue, v) <= 0) && (endValue == null || comp.compare(v, endValue) <= 0); } + abstract protected Comparator<T> getComparator(int colIdx); + public String toString() { return "LookupTable [path=" + table + "]"; } http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java index 980a263..3f67dcd 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java @@ -65,7 +65,7 @@ public class LookupTableTest extends LocalFileMetadataTestCase { for (String i : results) { System.out.println(i); - Assert.assertEquals("2012-01-01", i); + Assert.assertEquals(millis("2012-01-01"), i); } } @@ -77,11 +77,22 @@ public class LookupTableTest extends LocalFileMetadataTestCase { System.out.println("The first qtr_beg_dt is " + results.getFirst()); System.out.println("The last qtr_beg_dt is " + results.getSecond()); - Assert.assertEquals("2012-01-01", results.getFirst()); - Assert.assertEquals("2012-10-01", results.getSecond()); + Assert.assertEquals(millis("2012-01-01"), results.getFirst()); + Assert.assertEquals(millis("2012-10-01"), results.getSecond()); } @Test + public void testMapRange2() throws Exception { + Pair<String, String> results = lookupTable.mapRange("WEEK_BEG_DT", millis("2013-05-01"), millis("2013-08-01"), "CAL_DT"); + + System.out.println(DateFormat.formatToDateStr(Long.parseLong(results.getFirst()))); + System.out.println(DateFormat.formatToDateStr(Long.parseLong(results.getSecond()))); + + Assert.assertEquals(millis("2013-05-05"), results.getFirst()); + Assert.assertEquals(millis("2013-08-03"), results.getSecond()); + } + + @Test public void testMapValues() throws Exception { Set<String> values = new HashSet<String>(); values.add(millis("2012-01-24")); @@ -92,7 +103,7 @@ public class LookupTableTest extends LocalFileMetadataTestCase { for (String i : results) { System.out.println(i); - Assert.assertEquals("2012-01-01", i); + Assert.assertEquals(millis("2012-01-01"), i); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java index db3eb4f..61d87e8 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java @@ -21,13 +21,12 @@ package org.apache.kylin.metadata.filter; import java.nio.ByteBuffer; import java.util.Collection; import java.util.Collections; -import java.util.TreeSet; -import com.google.common.collect.Lists; -import org.apache.commons.collections.comparators.NullComparator; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.metadata.tuple.IEvaluatableTuple; +import com.google.common.collect.Lists; + /** * * @author xjiang http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/engine-mr/src/test/java/org/apache/kylin/engine/mr/LookupTableTest.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/LookupTableTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/LookupTableTest.java deleted file mode 100644 index 87f9133..0000000 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/LookupTableTest.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.engine.mr; - -import java.io.File; - -import org.apache.kylin.common.util.Array; -import org.apache.kylin.common.util.ByteArray; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.dict.lookup.LookupBytesTable; -import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.model.TableDesc; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * @author yangli9 - */ -public class LookupTableTest extends LocalFileMetadataTestCase { - - @Before - public void setup() throws Exception { - createTestMetadata(); - } - - @After - public void after() throws Exception { - cleanupTestMetadata(); - } - - @Test - public void testBasic() throws Exception { - TableDesc siteTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("EDW.TEST_SITES"); - TableDesc categoryTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("DEFAULT.test_category_groupings"); - LookupBytesTable lookup; - - System.out.println("============================================================================"); - - File f = new File(LOCALMETA_TEST_DATA + "/data/EDW.TEST_SITES.csv"); - lookup = new LookupBytesTable(siteTable, new String[] { "SITE_ID" }, new DFSFileTable("file://" + f.getAbsolutePath(), 10)); - lookup.dump(); - - System.out.println("============================================================================"); - - f = new File(LOCALMETA_TEST_DATA + "/data/DEFAULT.TEST_CATEGORY_GROUPINGS.csv"); - lookup = new LookupBytesTable(categoryTable, new String[] { "leaf_categ_id", "site_id" }, new DFSFileTable("file://" + f.getAbsolutePath(), 36)); - lookup.dump(); - - System.out.println("============================================================================"); - - ByteArray k1 = new ByteArray(Bytes.toBytes("533")); - ByteArray k2 = new ByteArray(Bytes.toBytes("0")); - Array<ByteArray> key = new Array<ByteArray>(new ByteArray[] { k1, k2 }); - System.out.println(lookup.getRow(key)); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/kylin-it/src/test/resources/query/sql_derived/query11.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_derived/query11.sql b/kylin-it/src/test/resources/query/sql_derived/query11.sql new file mode 100644 index 0000000..41157c5 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_derived/query11.sql @@ -0,0 +1,26 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select test_cal_dt.week_beg_dt, test_kylin_fact.lstg_format_name, test_category_groupings.meta_categ_name, sum(test_kylin_fact.price) as gmv, count(*) as trans_cnt + from test_kylin_fact + inner JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + inner JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + where test_cal_dt.week_beg_dt between DATE '2013-05-01' and DATE '2013-10-01' + group by test_cal_dt.week_beg_dt, test_kylin_fact.lstg_format_name, test_category_groupings.meta_categ_name http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java index 07f72b1..ef4ddf8 100644 --- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java +++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java @@ -24,11 +24,9 @@ import java.util.Properties; import org.apache.calcite.DataContext; import org.apache.calcite.jdbc.CalciteConnection; import org.apache.calcite.linq4j.Enumerator; -import org.apache.kylin.dict.DictCodeSystem; +import org.apache.kylin.common.util.DateFormat; import org.apache.kylin.metadata.filter.CompareTupleFilter; -import org.apache.kylin.metadata.filter.TimeConditionLiteralsReplacer; import org.apache.kylin.metadata.filter.TupleFilter; -import org.apache.kylin.metadata.filter.TupleFilterSerializer; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.ParameterDesc; @@ -112,11 +110,8 @@ public class OLAPEnumerator implements Enumerator<Object[]> { // bind dynamic variables bindVariable(olapContext.filter); - // modify date condition - olapContext.filter = modifyTimeLiterals(olapContext.filter); - olapContext.resetSQLDigest(); - // cube don't have correct result for simple query without group by, but let's try to return something makes sense + olapContext.resetSQLDigest(); SQLDigest sqlDigest = olapContext.getSQLDigest(); hackNoGroupByAggregation(sqlDigest); @@ -130,16 +125,6 @@ public class OLAPEnumerator implements Enumerator<Object[]> { return iterator; } - /** - * Calcite passed down all time family constants as GregorianCalendar, - * we'll have to reformat it to date/time according to column definition - */ - private TupleFilter modifyTimeLiterals(TupleFilter filter) { - TimeConditionLiteralsReplacer filterDecorator = new TimeConditionLiteralsReplacer(filter); - byte[] bytes = TupleFilterSerializer.serialize(filter, filterDecorator, DictCodeSystem.INSTANCE); - return TupleFilterSerializer.deserialize(bytes, DictCodeSystem.INSTANCE); - } - private void bindVariable(TupleFilter filter) { if (filter == null) { return; @@ -155,7 +140,11 @@ public class OLAPEnumerator implements Enumerator<Object[]> { String variable = entry.getKey(); Object value = optiqContext.get(variable); if (value != null) { - compFilter.bindVariable(variable, value.toString()); + String str = value.toString(); + if (compFilter.getColumn().getType().isDateTimeFamily()) + str = String.valueOf(DateFormat.stringToMillis(str)); + + compFilter.bindVariable(variable, str); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java index 2806b4b..ae6b265 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java @@ -50,7 +50,9 @@ import org.apache.calcite.rex.RexProgramBuilder; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.util.NlsString; +import org.apache.kylin.common.util.DateFormat; import org.apache.kylin.metadata.filter.CaseTupleFilter; import org.apache.kylin.metadata.filter.ColumnTupleFilter; import org.apache.kylin.metadata.filter.CompareTupleFilter; @@ -64,6 +66,7 @@ import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; import org.apache.kylin.metadata.model.TblColRef; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; /** @@ -144,7 +147,7 @@ public class OLAPFilterRel extends Filter implements OLAPRel { for (RexNode operand : call.operands) { TupleFilter childFilter = operand.accept(this); if (filter == null) { - filter = childFilter; + filter = cast(childFilter, call.type); } else { filter.addChild(childFilter); } @@ -159,6 +162,26 @@ public class OLAPFilterRel extends Filter implements OLAPRel { return filter; } + private TupleFilter cast(TupleFilter filter, RelDataType type) { + if ((filter instanceof ConstantTupleFilter) == false) { + return filter; + } + + ConstantTupleFilter constFilter = (ConstantTupleFilter) filter; + + if (type.getFamily() == SqlTypeFamily.DATE || type.getFamily() == SqlTypeFamily.DATETIME || type.getFamily() == SqlTypeFamily.TIMESTAMP) { + List<String> newValues = Lists.newArrayList(); + for (Object v : constFilter.getValues()) { + if (v == null) + newValues.add(null); + else + newValues.add(String.valueOf(DateFormat.stringToMillis(v.toString()))); + } + constFilter = new ConstantTupleFilter(newValues); + } + return constFilter; + } + private CompareTupleFilter mergeToInClause(TupleFilter filter) { List<? extends TupleFilter> children = filter.getChildren(); TblColRef inColumn = null; http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/storage-hbase/pom.xml ---------------------------------------------------------------------- diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml index 6c9e284..943b2d4 100644 --- a/storage-hbase/pom.xml +++ b/storage-hbase/pom.xml @@ -137,8 +137,10 @@ <include>org.apache.kylin:kylin-core-cube</include> <include>org.apache.kylin:kylin-invertedindex</include> <include>com.ning:compress-lzf</include> - <include>com.n3twork.druid:extendedset</include> <include>org.roaringbitmap:RoaringBitmap</include> + <!-- below for inverted index only --> + <include>com.n3twork.druid:extendedset</include> + <include>org.apache.commons:commons-lang3</include> </includes> </artifactSet> <filters> http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java index 4b3cb67..672bcbe 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java @@ -48,6 +48,7 @@ import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; import org.apache.kylin.cube.model.HBaseColumnDesc; import org.apache.kylin.cube.model.HBaseMappingDesc; +import org.apache.kylin.cube.model.RowKeyDesc; import org.apache.kylin.dict.lookup.LookupStringTable; import org.apache.kylin.measure.MeasureType; import org.apache.kylin.metadata.filter.ColumnTupleFilter; @@ -543,10 +544,13 @@ public class CubeStorageQuery implements IStorageQuery { } // a little pre-evaluation to remove invalid EQ/IN values and round start/end according to dictionary + RowKeyDesc rowkey = cubeSegment.getCubeDesc().getRowkey(); Iterator<ColumnValueRange> it = rangeMap.values().iterator(); while (it.hasNext()) { ColumnValueRange range = it.next(); - range.preEvaluateWithDict((Dictionary<String>) cubeSegment.getDictionary(range.getColumn())); + if (rowkey.isUseDictionary(range.getColumn())) { + range.preEvaluateWithDict((Dictionary<String>) cubeSegment.getDictionary(range.getColumn())); + } if (range.satisfyAll()) it.remove(); else if (range.satisfyNone())
