KYLIN-993 Support functions in where clause Signed-off-by: Li, Yang <yang...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bed15abb Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bed15abb Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bed15abb Branch: refs/heads/1.x-HBase1.1.3 Commit: bed15abbdd70305f5bd998eb7fb1775e6900fcee Parents: e9219d7 Author: lidongsjtu <don...@ebay.com> Authored: Mon Nov 16 22:39:56 2015 +0800 Committer: Li, Yang <yang...@ebay.com> Committed: Tue Nov 17 16:51:12 2015 +0800 ---------------------------------------------------------------------- .../metadata/filter/CompareTupleFilter.java | 23 +-- .../metadata/filter/FunctionTupleFilter.java | 154 +++++++++++++++++ .../metadata/filter/ITupleFilterTranslator.java | 26 +++ .../kylin/metadata/filter/TupleFilter.java | 2 +- .../metadata/filter/util/BuiltInMethod.java | 56 +++++++ .../kylin/query/relnode/OLAPFilterRel.java | 45 ++--- query/src/test/resources/query/sql/query85.sql | 22 +++ .../kylin/storage/hbase/CubeStorageEngine.java | 2 + .../hbase/coprocessor/CoprocessorFilter.java | 9 +- .../CoprocessorTupleFilterTranslator.java | 166 +++++++++++++++++++ 10 files changed, 457 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java ---------------------------------------------------------------------- diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java index d6368dd..6754ff7 100644 --- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java +++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java @@ -19,12 +19,9 @@ package org.apache.kylin.metadata.filter; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; +import java.util.*; +import org.apache.calcite.sql.SqlFunction; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.tuple.ITuple; @@ -39,6 +36,7 @@ public class CompareTupleFilter extends TupleFilter { private String firstCondValue; private Map<String, String> dynamicVariables; private String nullString; + private FunctionTupleFilter functionTupleFilter; public CompareTupleFilter(FilterOperatorEnum op) { super(new ArrayList<TupleFilter>(2), op); @@ -49,7 +47,7 @@ public class CompareTupleFilter extends TupleFilter { || op == FilterOperatorEnum.GT || op == FilterOperatorEnum.GTE // || op == FilterOperatorEnum.IN || op == FilterOperatorEnum.NOTIN // || op == FilterOperatorEnum.ISNULL || op == FilterOperatorEnum.ISNOTNULL); - if (opGood == false) + if (!opGood) throw new IllegalArgumentException("Unsupported operator " + op); } @@ -81,6 +79,8 @@ public class CompareTupleFilter extends TupleFilter { } else if (child instanceof DynamicTupleFilter) { DynamicTupleFilter dynamicFilter = (DynamicTupleFilter) child; this.dynamicVariables.put(dynamicFilter.getVariableName(), null); + } else if (child instanceof FunctionTupleFilter) { + this.functionTupleFilter = (FunctionTupleFilter)child; } //TODO // else if (child instanceof ExtractTupleFilter) { @@ -105,6 +105,10 @@ public class CompareTupleFilter extends TupleFilter { return column; } + public FunctionTupleFilter getFunctionTupleFilter() { + return functionTupleFilter; + } + public Map<String, String> getVariables() { return dynamicVariables; } @@ -137,7 +141,7 @@ public class CompareTupleFilter extends TupleFilter { @Override public String toString() { - return "CompareFilter [" + column + " " + operator + " " + conditionValues + ", children=" + children + "]"; + return "CompareFilter [" + (functionTupleFilter == null ? column : functionTupleFilter) + " " + operator + " " + conditionValues + ", children=" + children + "]"; } // TODO requires generalize, currently only evaluates COLUMN {op} CONST @@ -146,7 +150,7 @@ public class CompareTupleFilter extends TupleFilter { // extract tuple value String tupleValue = null; for (TupleFilter filter : this.children) { - if (isConstant(filter) == false) { + if (!isConstant(filter)) { filter.evaluate(tuple); tupleValue = filter.getValues().iterator().next(); } @@ -209,7 +213,7 @@ public class CompareTupleFilter extends TupleFilter { @Override public boolean isEvaluable() { - return column != null && !conditionValues.isEmpty(); + return (functionTupleFilter != null || column != null) && !conditionValues.isEmpty(); } @Override @@ -239,5 +243,4 @@ public class CompareTupleFilter extends TupleFilter { } this.nullString = BytesUtil.readAsciiString(buffer); } - } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java ---------------------------------------------------------------------- diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java new file mode 100644 index 0000000..62ab42f --- /dev/null +++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.metadata.filter; + +import com.google.common.collect.Lists; +import com.google.common.primitives.Primitives; +import org.apache.calcite.sql.SqlOperator; +import org.apache.kylin.metadata.filter.util.BuiltInMethod; +import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.metadata.tuple.ITuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Collection; +import java.util.List; + +/** + * Created by dongli on 11/11/15. + */ +public class FunctionTupleFilter extends TupleFilter { + public static final Logger logger = LoggerFactory.getLogger(FunctionTupleFilter.class); + + private SqlOperator sqlOperator; + // FIXME Only supports single parameter functions currently + private TupleFilter columnContainerFilter; + private int colPosition; + private Method method; + private List<Object> methodParams; + private boolean isValid = false; + + public FunctionTupleFilter(SqlOperator sqlOperator) { + super(Lists.<TupleFilter>newArrayList(), FilterOperatorEnum.FUNCTION); + this.methodParams = Lists.newArrayList(); + this.sqlOperator = sqlOperator; + + String opName = sqlOperator.getName().toUpperCase(); + if (BuiltInMethod.MAP.containsKey(opName)) { + this.method = BuiltInMethod.MAP.get(opName).method; + isValid = true; + } + } + + public SqlOperator getSqlOperator() { + return sqlOperator; + } + + public TblColRef getColumn() { + if (columnContainerFilter == null) + return null; + + if (columnContainerFilter instanceof ColumnTupleFilter) + return ((ColumnTupleFilter)columnContainerFilter).getColumn(); + else if (columnContainerFilter instanceof FunctionTupleFilter) + return ((FunctionTupleFilter)columnContainerFilter).getColumn(); + + throw new UnsupportedOperationException("Wrong type TupleFilter in FunctionTupleFilter."); + } + + public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException { + if (columnContainerFilter instanceof ColumnTupleFilter) + methodParams.set(colPosition, input); + else if (columnContainerFilter instanceof FunctionTupleFilter) + methodParams.set(colPosition, ((FunctionTupleFilter) columnContainerFilter).invokeFunction(input)); + return method.invoke(null, (Object[])(methodParams.toArray())); + } + + public boolean isValid() { + return isValid && method != null && methodParams.size() == children.size(); + } + + @Override + @SuppressWarnings("unchecked") + public void addChild(TupleFilter child) { + if (child instanceof ColumnTupleFilter || child instanceof FunctionTupleFilter) { + columnContainerFilter = child; + colPosition = methodParams.size(); + methodParams.add(null); + } else if (child instanceof ConstantTupleFilter) { + String constVal = child.getValues().iterator().next(); + try { + Class clazz = Primitives.wrap(method.getParameterTypes()[methodParams.size()]); + if (!Primitives.isWrapperType(clazz)) + methodParams.add(constVal); + else + methodParams.add(clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, constVal))); + } catch (Exception e) { + logger.debug(e.getMessage()); + isValid = false; + } + } + super.addChild(child); + } + + @Override + public boolean isEvaluable() { + return false; + } + + @Override + public boolean evaluate(ITuple tuple) { + throw new UnsupportedOperationException("Function filter cannot be evaluated immediately"); + } + + @Override + public Collection<String> getValues() { + throw new UnsupportedOperationException("Function filter cannot be evaluated immediately"); + } + + @Override + public String toString(){ + StringBuilder sb = new StringBuilder(); + sb.append(sqlOperator.getName()); + sb.append("("); + for (int i = 0; i < methodParams.size(); i++) { + if (colPosition == i) { + sb.append(columnContainerFilter); + } else { + sb.append(methodParams.get(i)); + } + if (i < methodParams.size() - 1) + sb.append(","); + } + sb.append(")"); + return sb.toString(); + } + + @Override + public byte[] serialize() { + throw new UnsupportedOperationException("Method serialize() is not supported for FunctionTupleFilter."); + } + + @Override + public void deserialize(byte[] bytes) { + throw new UnsupportedOperationException("Method deserialize(byte[] bytes) is not supported for FunctionTupleFilter."); + } +} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java ---------------------------------------------------------------------- diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java new file mode 100644 index 0000000..7f1aed6 --- /dev/null +++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.metadata.filter; + +/** + * Created by dongli on 11/11/15. + */ +public interface ITupleFilterTranslator { + TupleFilter translate(TupleFilter tupleFilter); +} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java ---------------------------------------------------------------------- diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java index 66dc0db..25cae20 100644 --- a/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java +++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java @@ -37,7 +37,7 @@ import com.google.common.collect.Maps; public abstract class TupleFilter { public enum FilterOperatorEnum { - EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34); + EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35); private final int value; http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/metadata/src/main/java/org/apache/kylin/metadata/filter/util/BuiltInMethod.java ---------------------------------------------------------------------- diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/util/BuiltInMethod.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/util/BuiltInMethod.java new file mode 100644 index 0000000..1f15c9c --- /dev/null +++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/util/BuiltInMethod.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.metadata.filter.util; + +import com.google.common.collect.ImmutableMap; +import org.apache.calcite.avatica.util.DateTimeUtils; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.runtime.SqlFunctions; + +import java.lang.reflect.Method; + +/** + * Created by dongli on 11/13/15. + */ +public enum BuiltInMethod { + UPPER(SqlFunctions.class, "upper", String.class), + LOWER(SqlFunctions.class, "lower", String.class), + SUBSTRING(SqlFunctions.class, "substring", String.class, int.class, int.class), + CHAR_LENGTH(SqlFunctions.class, "charLength", String.class), + LIKE(SqlFunctions.class, "like", String.class, String.class), + INITCAP(SqlFunctions.class, "initcap", String.class); + public final Method method; + + public static final ImmutableMap<String, BuiltInMethod> MAP; + + static { + final ImmutableMap.Builder<String, BuiltInMethod> builder = + ImmutableMap.builder(); + for (BuiltInMethod value : BuiltInMethod.values()) { + if (value.method != null) { + builder.put(value.name(), value); + } + } + MAP = builder.build(); + } + + BuiltInMethod(Class clazz, String methodName, Class... argumentTypes) { + this.method = Types.lookupMethod(clazz, methodName, argumentTypes); + } +} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java index cfd5e3b..583af9a 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java @@ -18,50 +18,25 @@ package org.apache.kylin.query.relnode; -import java.util.Calendar; -import java.util.GregorianCalendar; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; - +import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; import org.apache.calcite.adapter.enumerable.EnumerableCalc; import org.apache.calcite.adapter.enumerable.EnumerableConvention; import org.apache.calcite.adapter.enumerable.EnumerableRel; import org.apache.calcite.avatica.util.TimeUnitRange; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptCost; -import org.apache.calcite.plan.RelOptPlanner; -import org.apache.calcite.plan.RelTrait; -import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.plan.*; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexDynamicParam; -import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexLiteral; -import org.apache.calcite.rex.RexLocalRef; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexProgram; -import org.apache.calcite.rex.RexProgramBuilder; -import org.apache.calcite.rex.RexVisitorImpl; +import org.apache.calcite.rex.*; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.util.NlsString; -import org.apache.kylin.metadata.filter.CaseTupleFilter; -import org.apache.kylin.metadata.filter.ColumnTupleFilter; -import org.apache.kylin.metadata.filter.CompareTupleFilter; -import org.apache.kylin.metadata.filter.ConstantTupleFilter; -import org.apache.kylin.metadata.filter.DynamicTupleFilter; -import org.apache.kylin.metadata.filter.ExtractTupleFilter; -import org.apache.kylin.metadata.filter.LogicalTupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter; +import org.apache.kylin.metadata.filter.*; import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; import org.apache.kylin.metadata.model.TblColRef; -import com.google.common.base.Preconditions; -import com.google.common.collect.Sets; +import java.util.*; /** */ @@ -127,9 +102,13 @@ public class OLAPFilterRel extends Filter implements OLAPRel { if (op.getName().equalsIgnoreCase("extract_date")) { filter = new ExtractTupleFilter(FilterOperatorEnum.EXTRACT); } else { - throw new UnsupportedOperationException(op.getName()); + filter = new FunctionTupleFilter(op); } break; + case LIKE: + case OTHER_FUNCTION: + filter = new FunctionTupleFilter(op); + break; default: throw new UnsupportedOperationException(op.getName()); } @@ -286,6 +265,8 @@ public class OLAPFilterRel extends Filter implements OLAPRel { } private void collectColumnsRecursively(TupleFilter filter, Set<TblColRef> collector) { + if (filter == null) return; + if (filter instanceof ColumnTupleFilter) { collector.add(((ColumnTupleFilter) filter).getColumn()); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/query/src/test/resources/query/sql/query85.sql ---------------------------------------------------------------------- diff --git a/query/src/test/resources/query/sql/query85.sql b/query/src/test/resources/query/sql/query85.sql new file mode 100644 index 0000000..4e80d59 --- /dev/null +++ b/query/src/test/resources/query/sql/query85.sql @@ -0,0 +1,22 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select upper(lstg_format_name) as lstg_format_name, count(*) as cnt from test_kylin_fact +where lower(lstg_format_name)='abin' and substring(lstg_format_name,1,3) in ('ABI') and upper(lstg_format_name) > 'AAAA' and +lower(lstg_format_name) like '%b%' and char_length(lstg_format_name) < 10 and char_length(lstg_format_name) > 3 and lstg_format_name||'a'='ABINa' +group by lstg_format_name \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java index 8eb7bcb..fdb8986 100644 --- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java +++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java @@ -267,6 +267,8 @@ public class CubeStorageEngine implements IStorageEngine { } private void collectColumnsRecursively(TupleFilter filter, Set<TblColRef> collector) { + if (filter == null) return; + if (filter instanceof ColumnTupleFilter) { collectColumns(((ColumnTupleFilter) filter).getColumn(), collector); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java index 65fddd2..b3e2d31 100644 --- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java +++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java @@ -26,11 +26,7 @@ import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.cube.kv.RowKeyColumnIO; import org.apache.kylin.dict.Dictionary; import org.apache.kylin.dict.ISegment; -import org.apache.kylin.metadata.filter.ColumnTupleFilter; -import org.apache.kylin.metadata.filter.CompareTupleFilter; -import org.apache.kylin.metadata.filter.ConstantTupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter; -import org.apache.kylin.metadata.filter.TupleFilterSerializer; +import org.apache.kylin.metadata.filter.*; import org.apache.kylin.metadata.filter.TupleFilterSerializer.Decorator; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.tuple.ITuple; @@ -61,6 +57,9 @@ public class CoprocessorFilter { if (filter == null) return null; + ITupleFilterTranslator translator = new CoprocessorTupleFilterTranslator(columnIO); + filter = translator.translate(filter); + // un-evaluatable filter is replaced with TRUE if (!filter.isEvaluable()) { TupleFilter.collectColumns(filter, unstrictlyFilteredColumns); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bed15abb/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorTupleFilterTranslator.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorTupleFilterTranslator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorTupleFilterTranslator.java new file mode 100644 index 0000000..aae945d --- /dev/null +++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorTupleFilterTranslator.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.storage.hbase.coprocessor; + +import com.google.common.primitives.Primitives; +import org.apache.kylin.cube.kv.RowKeyColumnIO; +import org.apache.kylin.dict.Dictionary; +import org.apache.kylin.metadata.filter.*; +import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; +import org.apache.kylin.metadata.model.TblColRef; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; + +/** + * Created by dongli on 11/11/15. + */ +public class CoprocessorTupleFilterTranslator implements ITupleFilterTranslator { + public static final Logger logger = LoggerFactory.getLogger(CoprocessorTupleFilterTranslator.class); + + private RowKeyColumnIO rowKeyColumnIO; + + public CoprocessorTupleFilterTranslator(RowKeyColumnIO rowKeyColumnIO) { + this.rowKeyColumnIO = rowKeyColumnIO; + } + + @Override + public TupleFilter translate(TupleFilter tupleFilter) { + TupleFilter translated = null; + if (tupleFilter instanceof CompareTupleFilter) { + logger.info("Translation to IN clause: " + tupleFilter); + translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter); + logger.info(translated == null ? "Failed, will use Calcite to handle computed comparison." : "Succeed: " + translated); + } else if (tupleFilter instanceof FunctionTupleFilter) { + logger.info("Translation to IN clause: " + tupleFilter); + translated = translateFunctionTupleFilter((FunctionTupleFilter) tupleFilter); + logger.info(translated == null ? "Failed, will use Calcite to handle computed column." : "Succeed: " + translated); + } + return translated == null ? tupleFilter : translated; + } + + private TupleFilter translateFunctionTupleFilter(FunctionTupleFilter functionTupleFilter) { + if (!functionTupleFilter.isValid()) + return null; + + TblColRef columnRef = functionTupleFilter.getColumn(); + Dictionary<String> dict = rowKeyColumnIO.getDictionary(columnRef); + if (dict == null) + return null; + + CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); + translated.addChild(new ColumnTupleFilter(columnRef)); + + try { + for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { + String dictVal = dict.getValueFromId(i); + if ((Boolean)functionTupleFilter.invokeFunction(dictVal)) { + translated.addChild(new ConstantTupleFilter(dictVal)); + } + } + } catch (IllegalAccessException e) { + logger.debug(e.getMessage()); + return null; + } catch (InvocationTargetException e) { + logger.debug(e.getMessage()); + return null; + } catch (IllegalArgumentException e) { + logger.debug(e.getMessage()); + return null; + } + return translated; + } + + @SuppressWarnings("unchecked") + private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter){ + if (compTupleFilter.getFunctionTupleFilter() == null) + return null; + + FunctionTupleFilter functionTupleFilter = compTupleFilter.getFunctionTupleFilter(); + if (!functionTupleFilter.isValid()) + return null; + + TblColRef columnRef = functionTupleFilter.getColumn(); + Dictionary<String> dict = rowKeyColumnIO.getDictionary(columnRef); + if (dict == null) + return null; + + CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); + translated.addChild(new ColumnTupleFilter(columnRef)); + + try { + for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { + String dictVal = dict.getValueFromId(i); + Object computedVal = functionTupleFilter.invokeFunction(dictVal); + Class clazz = Primitives.wrap(computedVal.getClass()); + Object targetVal = compTupleFilter.getFirstValue(); + if (Primitives.isWrapperType(clazz)) + targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue())); + + int comp = ((Comparable)computedVal).compareTo(targetVal); + boolean compResult = false; + switch (compTupleFilter.getOperator()) { + case EQ: + compResult = comp == 0; + break; + case NEQ: + compResult = comp != 0; + break; + case LT: + compResult = comp < 0; + break; + case LTE: + compResult = comp <= 0; + break; + case GT: + compResult = comp > 0; + break; + case GTE: + compResult = comp >= 0; + break; + case IN: + compResult = compTupleFilter.getValues().contains(computedVal.toString()); + break; + case NOTIN: + compResult = !compTupleFilter.getValues().contains(computedVal.toString()); + break; + default: + break; + } + if (compResult) { + translated.addChild(new ConstantTupleFilter(dictVal)); + } + } + } catch (IllegalAccessException e) { + logger.debug(e.getMessage()); + return null; + } catch (InvocationTargetException e) { + logger.debug(e.getMessage()); + return null; + } catch (IllegalArgumentException e) { + logger.debug(e.getMessage()); + return null; + } catch (NoSuchMethodException e) { + logger.debug(e.getMessage()); + return null; + } + return translated; + } +}
