KYLIN-2254 allow 2 columns in a CompareTupleFilter
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/350fad8f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/350fad8f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/350fad8f Branch: refs/heads/master-hbase1.x Commit: 350fad8f248988596c05b080b3c5a68b77b790a1 Parents: b1fd467 Author: Li Yang <[email protected]> Authored: Wed Dec 7 17:51:26 2016 +0800 Committer: Li Yang <[email protected]> Committed: Thu Dec 8 09:47:27 2016 +0800 ---------------------------------------------------------------------- .../metadata/filter/CompareTupleFilter.java | 23 ++++++------ .../kylin/metadata/model/FunctionDesc.java | 2 +- .../apache/kylin/metadata/model/TblColRef.java | 2 +- .../resources/query/sql_subquery/query12.sql | 37 ++++++++++++++++++++ .../kylin/query/relnode/OLAPAggregateRel.java | 13 ++++--- .../kylin/query/relnode/OLAPFilterRel.java | 30 +++++----------- 6 files changed, 66 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java index f2af735..d567c89 100755 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java @@ -37,6 +37,7 @@ public class CompareTupleFilter extends TupleFilter { // operand 1 is either a column or a function private TblColRef column; private FunctionTupleFilter function; + private TblColRef secondColumn; // operand 2 is constants private Set<Object> conditionValues; @@ -73,15 +74,16 @@ public class CompareTupleFilter extends TupleFilter { if (child instanceof ColumnTupleFilter) { ColumnTupleFilter columnFilter = (ColumnTupleFilter) child; if (this.column != null) { - throw new IllegalStateException("Duplicate columns! old is " + column.getName() + " and new is " + columnFilter.getColumn().getName()); - } - this.column = columnFilter.getColumn(); - // if value is before column, we need to reverse the operator. e.g. "1 >= c1" => "c1 <= 1" - // children.size() > 1 means already added one conditionValue or dynamicVariable - if (this.children.size() > 1 && needSwapOperator()) { - this.operator = SWAP_OP_MAP.get(this.operator); - TupleFilter last = this.children.remove(this.children.size() - 1); - this.children.add(0, last); + this.secondColumn = columnFilter.getColumn(); + } else { + this.column = columnFilter.getColumn(); + // if value is before column, we need to reverse the operator. e.g. "1 >= c1" => "c1 <= 1" + // children.size() > 1 means already added one conditionValue or dynamicVariable + if (this.children.size() > 1 && needSwapOperator()) { + this.operator = SWAP_OP_MAP.get(this.operator); + TupleFilter last = this.children.remove(this.children.size() - 1); + this.children.add(0, last); + } } } else if (child instanceof ConstantTupleFilter) { this.conditionValues.addAll(child.getValues()); @@ -212,7 +214,8 @@ public class CompareTupleFilter extends TupleFilter { @Override public boolean isEvaluable() { - return ((function != null && function.isEvaluable()) || column != null) && !conditionValues.isEmpty(); + return (column != null || (function != null && function.isEvaluable())) // + && !conditionValues.isEmpty() && secondColumn == null; } @SuppressWarnings({ "unchecked", "rawtypes" }) http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java index 832cb4a..9252c42 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java @@ -140,7 +140,7 @@ public class FunctionDesc { } else if (isCount()) { return "COUNT__"; // ignores parameter, count(*), count(1), count(col) are all the same } else { - return getFullExpression().replaceAll("[(), ]", "_"); + return getFullExpression().replaceAll("[(),. ]", "_"); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java index 5d72c3f..fd0224f 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java @@ -150,7 +150,7 @@ public class TblColRef implements Serializable { return column.getType(); } - public void markInnerColumn(InnerDataTypeEnum dataType) { + private void markInnerColumn(InnerDataTypeEnum dataType) { this.column.setDatatype(dataType.getDataType()); this.column.getTable().setName(INNER_TABLE_NAME); this.column.getTable().setDatabase("DEFAULT"); http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/kylin-it/src/test/resources/query/sql_subquery/query12.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_subquery/query12.sql b/kylin-it/src/test/resources/query/sql_subquery/query12.sql new file mode 100644 index 0000000..6ed45fa --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_subquery/query12.sql @@ -0,0 +1,37 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +SELECT + f.lstg_format_name + ,sum(price) as sum_price +FROM + test_kylin_fact f + inner join + ( + select + lstg_format_name, + min(slr_segment_cd) as min_seg + from + test_kylin_fact + group by + lstg_format_name + ) t on f.lstg_format_name = t.lstg_format_name +where + f.slr_segment_cd = min_seg +group by + f.lstg_format_name http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java index 24933f5..f5fa74d 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java @@ -192,17 +192,16 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel { for (int i = 0; i < this.aggregations.size(); i++) { FunctionDesc aggFunc = this.aggregations.get(i); - TblColRef aggCol = null; + String aggOutName; if (aggFunc.needRewriteField()) { - aggCol = buildRewriteColumn(aggFunc); + aggOutName = aggFunc.getRewriteFieldName(); } else { AggregateCall aggCall = this.rewriteAggCalls.get(i); - if (!aggCall.getArgList().isEmpty()) { - int index = aggCall.getArgList().get(0); - aggCol = inputColumnRowType.getColumnByIndex(index); - } + int index = aggCall.getArgList().get(0); + aggOutName = aggFunc.getExpression() + "_" + inputColumnRowType.getColumnByIndex(index).getIdentity() + "_"; } - columns.add(aggCol); + TblColRef aggOutCol = TblColRef.newInnerColumn(aggOutName, TblColRef.InnerDataTypeEnum.LITERAL); + columns.add(aggOutCol); } return new ColumnRowType(columns); } http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java index 411142d..8023df4 100755 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java @@ -20,6 +20,7 @@ package org.apache.kylin.query.relnode; import java.util.GregorianCalendar; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -61,14 +62,13 @@ import org.apache.kylin.metadata.filter.DynamicTupleFilter; import org.apache.kylin.metadata.filter.ExtractTupleFilter; import org.apache.kylin.metadata.filter.LogicalTupleFilter; import org.apache.kylin.metadata.filter.TupleFilter; -import org.apache.kylin.metadata.filter.UnsupportedTupleFilter; import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; +import org.apache.kylin.metadata.filter.UnsupportedTupleFilter; import org.apache.kylin.metadata.filter.function.Functions; import org.apache.kylin.metadata.model.TblColRef; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; /** */ @@ -78,6 +78,7 @@ public class OLAPFilterRel extends Filter implements OLAPRel { private final ColumnRowType inputRowType; private final OLAPContext context; + private final Set<TblColRef> columnsInFilter = new HashSet<>(); public TupleFilterVisitor(ColumnRowType inputRowType, OLAPContext context) { super(true); @@ -227,7 +228,10 @@ public class OLAPFilterRel extends Filter implements OLAPRel { @Override public TupleFilter visitInputRef(RexInputRef inputRef) { TblColRef column = inputRowType.getColumnByIndex(inputRef.getIndex()); - context.allColumns.add(column); + if (!column.isInnerColumn()) { + context.allColumns.add(column); + columnsInFilter.add(column); + } ColumnTupleFilter filter = new ColumnTupleFilter(column); return filter; } @@ -320,25 +324,7 @@ public class OLAPFilterRel extends Filter implements OLAPRel { TupleFilterVisitor visitor = new TupleFilterVisitor(this.columnRowType, context); context.filter = this.condition.accept(visitor); - context.filterColumns = collectColumns(context.filter); - } - - private Set<TblColRef> collectColumns(TupleFilter filter) { - Set<TblColRef> ret = Sets.newHashSet(); - collectColumnsRecursively(filter, ret); - return ret; - } - - private void collectColumnsRecursively(TupleFilter filter, Set<TblColRef> collector) { - if (filter == null) - return; - - if (filter instanceof ColumnTupleFilter) { - collector.add(((ColumnTupleFilter) filter).getColumn()); - } - for (TupleFilter child : filter.getChildren()) { - collectColumnsRecursively(child, collector); - } + context.filterColumns = visitor.columnsInFilter; } @Override
