This is an automated email from the ASF dual-hosted git repository. volodymyr pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
commit c230ba55cceb6d48ac9c1ab0701a167d91842a11 Author: Volodymyr Vysotskyi <[email protected]> AuthorDate: Tue Jan 22 00:18:19 2019 +0200 DRILL-6533: Allow using literal values in functions which expect FieldReader instead of ValueHolder closes #1617 --- .../expr/fn/interpreter/InterpreterEvaluator.java | 50 +++++--- .../drill/exec/expr/fn/impl/TestTypeFns.java | 133 +++++++++++++++++++-- .../main/codegen/templates/BasicTypeHelper.java | 30 ++++- 3 files changed, 187 insertions(+), 26 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterEvaluator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterEvaluator.java index a0373d9..7648ff4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterEvaluator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterEvaluator.java @@ -24,6 +24,7 @@ import java.util.Objects; import javax.annotation.Nullable; import javax.inject.Inject; +import org.apache.drill.exec.expr.BasicTypeHelper; import org.apache.drill.shaded.guava.com.google.common.base.Function; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.expression.BooleanOperator; @@ -89,7 +90,17 @@ public class InterpreterEvaluator { } - public static ValueHolder evaluateFunction(DrillSimpleFunc interpreter, ValueHolder[] args, String funcName) throws Exception { + /** + * Assigns specified {@code Object[] args} to the function arguments, + * evaluates function and returns its result. + * + * @param interpreter function to be evaluated + * @param args function arguments + * @param funcName name of the function + * @return result of function call stored in {@link ValueHolder} + * @throws Exception if {@code args} types does not match function input arguments types + */ + public static ValueHolder evaluateFunction(DrillSimpleFunc interpreter, Object[] args, String funcName) throws Exception { Preconditions.checkArgument(interpreter != null, "interpreter could not be null when use interpreted model to evaluate function " + funcName); // the current input index to assign into the next available parameter, found using the @Param notation @@ -100,13 +111,13 @@ public class InterpreterEvaluator { Field[] fields = interpreter.getClass().getDeclaredFields(); for (Field f : fields) { // if this is annotated as a parameter to the function - if ( f.getAnnotation(Param.class) != null ) { + if (f.getAnnotation(Param.class) != null) { f.setAccessible(true); if (currParameterIndex < args.length) { f.set(interpreter, args[currParameterIndex]); } currParameterIndex++; - } else if ( f.getAnnotation(Output.class) != null ) { + } else if (f.getAnnotation(Output.class) != null) { f.setAccessible(true); outField = f; // create an instance of the holder for the output to be stored in @@ -127,9 +138,8 @@ public class InterpreterEvaluator { } interpreter.setup(); interpreter.eval(); - ValueHolder out = (ValueHolder) outField.get(interpreter); - return out; + return (ValueHolder) outField.get(interpreter); } private static class InitVisitor extends AbstractExprVisitor<LogicalExpression, VectorAccessible, RuntimeException> { @@ -307,25 +317,36 @@ public class InterpreterEvaluator { DrillSimpleFuncHolder holder = (DrillSimpleFuncHolder) holderExpr.getHolder(); - ValueHolder [] args = new ValueHolder [holderExpr.args.size()]; + // function arguments may have different types: + // usually ValueHolder inheritors but sometimes FieldReader ones + Object[] args = new Object[holderExpr.args.size()]; for (int i = 0; i < holderExpr.args.size(); i++) { - args[i] = holderExpr.args.get(i).accept(this, inIndex); + ValueHolder valueHolder = holderExpr.args.get(i).accept(this, inIndex); + Object resultArg = valueHolder; + TypeProtos.MajorType argType = TypeHelper.getValueHolderType(valueHolder); + TypeProtos.MajorType holderParamType = holder.getParameters()[i].getType(); // In case function use "NULL_IF_NULL" policy. if (holder.getNullHandling() == FunctionTemplate.NullHandling.NULL_IF_NULL) { // Case 1: parameter is non-nullable, argument is nullable. - if (holder.getParameters()[i].getType().getMode() == TypeProtos.DataMode.REQUIRED && TypeHelper.getValueHolderType(args[i]).getMode() == TypeProtos.DataMode.OPTIONAL) { + if (holderParamType.getMode() == TypeProtos.DataMode.REQUIRED + && argType.getMode() == TypeProtos.DataMode.OPTIONAL) { // Case 1.1 : argument is null, return null value holder directly. - if (TypeHelper.isNull(args[i])) { + if (TypeHelper.isNull(valueHolder)) { return TypeHelper.createValueHolder(holderExpr.getMajorType()); } else { // Case 1.2: argument is nullable but not null value, deNullify it. - args[i] = TypeHelper.deNullify(args[i]); + resultArg = TypeHelper.deNullify(valueHolder); } - } else if (holder.getParameters()[i].getType().getMode() == TypeProtos.DataMode.OPTIONAL && TypeHelper.getValueHolderType(args[i]).getMode() == TypeProtos.DataMode.REQUIRED) { + } else if (holderParamType.getMode() == TypeProtos.DataMode.OPTIONAL + && argType.getMode() == TypeProtos.DataMode.REQUIRED) { // Case 2: parameter is nullable, argument is non-nullable. Nullify it. - args[i] = TypeHelper.nullify(args[i]); + resultArg = TypeHelper.nullify(valueHolder); } } + if (holder.getParameters()[i].isFieldReader()) { + resultArg = BasicTypeHelper.getHolderReaderImpl(argType, valueHolder); + } + args[i] = resultArg; } try { @@ -333,10 +354,11 @@ public class InterpreterEvaluator { ValueHolder out = evaluateFunction(interpreter, args, holderExpr.getName()); - if (TypeHelper.getValueHolderType(out).getMode() == TypeProtos.DataMode.OPTIONAL && + TypeProtos.MajorType outputType = TypeHelper.getValueHolderType(out); + if (outputType.getMode() == TypeProtos.DataMode.OPTIONAL && holderExpr.getMajorType().getMode() == TypeProtos.DataMode.REQUIRED) { return TypeHelper.deNullify(out); - } else if (TypeHelper.getValueHolderType(out).getMode() == TypeProtos.DataMode.REQUIRED && + } else if (outputType.getMode() == TypeProtos.DataMode.REQUIRED && holderExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) { return TypeHelper.nullify(out); } else { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestTypeFns.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestTypeFns.java index 02d664f..46e3216 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestTypeFns.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestTypeFns.java @@ -75,19 +75,19 @@ public class TestTypeFns extends ClusterTest { // typeof() returns types using the internal names. String sql = "SELECT typeof(CAST(a AS " + castType + ")) FROM (VALUES (1)) AS T(a)"; - String result = client.queryBuilder().sql(sql).singletonString(); + String result = queryBuilder().sql(sql).singletonString(); assertEquals(resultType, result); // For typeof(), null values annoyingly report a type of "NULL" sql = "SELECT typeof(CAST(a AS " + castType + ")) FROM cp.`functions/null.json`"; - result = client.queryBuilder().sql(sql).singletonString(); + result = queryBuilder().sql(sql).singletonString(); assertEquals("NULL", result); } private void doTypeOfTestSpecial(String expr, String value, String resultType) throws RpcException { String sql = "SELECT typeof(" + expr + ") FROM (VALUES (" + value + ")) AS T(a)"; - String result = client.queryBuilder().sql(sql).singletonString(); + String result = queryBuilder().sql(sql).singletonString(); assertEquals(resultType, result); } @@ -124,19 +124,25 @@ public class TestTypeFns extends ClusterTest { // sqlTypeOf() returns SQL type names: the names used in CAST. String sql = "SELECT sqlTypeOf(CAST(a AS " + type + ")) FROM (VALUES (1)) AS T(a)"; - String result = client.queryBuilder().sql(sql).singletonString(); + String result = queryBuilder().sql(sql).singletonString(); + assertEquals(type, result); + + // sqlTypeOf() returns SQL type names: the names used in CAST. + + sql = "SELECT sqlTypeOf(CAST(1 AS " + type + "))"; + result = queryBuilder().sql(sql).singletonString(); assertEquals(type, result); // Returns same type even value is null. sql = "SELECT sqlTypeOf(CAST(a AS " + type + ")) FROM cp.`functions/null.json`"; - result = client.queryBuilder().sql(sql).singletonString(); + result = queryBuilder().sql(sql).singletonString(); assertEquals(type, result); } private void doSqlTypeOfTestSpecial(String expr, String value, String resultType) throws RpcException { String sql = "SELECT sqlTypeof(" + expr + ") FROM (VALUES (" + value + ")) AS T(a)"; - String result = client.queryBuilder().sql(sql).singletonString(); + String result = queryBuilder().sql(sql).singletonString(); assertEquals(resultType, result); } @@ -163,13 +169,17 @@ public class TestTypeFns extends ClusterTest { // drillTypeOf() returns types using the internal names. String sql = "SELECT drillTypeOf(CAST(a AS " + castType + ")) FROM (VALUES (1)) AS T(a)"; - String result = client.queryBuilder().sql(sql).singletonString(); + String result = queryBuilder().sql(sql).singletonString(); + assertEquals(resultType, result); + + sql = "SELECT drillTypeOf(CAST(1 AS " + castType + "))"; + result = queryBuilder().sql(sql).singletonString(); assertEquals(resultType, result); // Returns same type even value is null. sql = "SELECT drillTypeOf(CAST(a AS " + castType + ")) FROM cp.`functions/null.json`"; - result = client.queryBuilder().sql(sql).singletonString(); + result = queryBuilder().sql(sql).singletonString(); assertEquals(resultType, result); } @@ -179,19 +189,120 @@ public class TestTypeFns extends ClusterTest { // CSV files with headers use REQUIRED mode String sql = "SELECT modeOf(`name`) FROM cp.`store/text/data/cars.csvh`"; - String result = client.queryBuilder().sql(sql).singletonString(); + String result = queryBuilder().sql(sql).singletonString(); assertEquals("NOT NULL", result); // CSV files without headers use REPEATED mode sql = "SELECT modeOf(`columns`) FROM cp.`textinput/input2.csv`"; - result = client.queryBuilder().sql(sql).singletonString(); + result = queryBuilder().sql(sql).singletonString(); assertEquals("ARRAY", result); // JSON files use OPTIONAL mode sql = "SELECT modeOf(`name`) FROM cp.`jsoninput/specialchar.json`"; - result = client.queryBuilder().sql(sql).singletonString(); + result = queryBuilder().sql(sql).singletonString(); assertEquals("NULLABLE", result); } + + @Test + public void testTypeOfLiteral() throws Exception { + String sql = + "SELECT typeOf(1) c1," + + "typeOf('a') c2," + + "typeOf(date '2018-01-22') c3," + + "typeOf(time '01:00:20.123') c4," + + "typeOf(timestamp '2018-01-22 01:00:20.123') c5," + + "typeOf(false) c6," + + "typeOf(12.3) c7," + + "typeOf(1>2) c8," + + "typeOf(cast(null as int)) c9"; + + testBuilder() + .sqlQuery(sql) + .unOrdered() + .baselineColumns("c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9") + .baselineValues("INT", "VARCHAR", "DATE", "TIME", "TIMESTAMP", "BIT", "VARDECIMAL", "BIT", "NULL") + .go(); + } + + @Test + public void testSqlTypeOfLiteral() throws Exception { + String sql = + "SELECT sqlTypeOf(1) c1," + + "sqlTypeOf('a') c2," + + "sqlTypeOf(date '2018-01-22') c3," + + "sqlTypeOf(time '01:00:20.123') c4," + + "sqlTypeOf(timestamp '2018-01-22 01:00:20.123') c5," + + "sqlTypeOf(false) c6," + + "sqlTypeOf(12.3) c7," + + "sqlTypeOf(1>2) c8," + + "sqlTypeOf(cast(null as int)) c9"; + + testBuilder() + .sqlQuery(sql) + .unOrdered() + .baselineColumns("c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9") + .baselineValues("INTEGER", "CHARACTER VARYING", "DATE", "TIME", + "TIMESTAMP", "BOOLEAN", "DECIMAL(3, 1)", "BOOLEAN", "INTEGER") + .go(); + } + + @Test + public void testDrillTypeOfLiteral() throws Exception { + String sql = + "SELECT drillTypeOf(1) c1," + + "drillTypeOf('a') c2," + + "drillTypeOf(date '2018-01-22') c3," + + "drillTypeOf(time '01:00:20.123') c4," + + "drillTypeOf(timestamp '2018-01-22 01:00:20.123') c5," + + "drillTypeOf(false) c6," + + "drillTypeOf(12.3) c7," + + "drillTypeOf(1>2) c8," + + "drillTypeOf(cast(null as int)) c9"; + + testBuilder() + .sqlQuery(sql) + .unOrdered() + .baselineColumns("c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9") + .baselineValues("INT", "VARCHAR", "DATE", "TIME", + "TIMESTAMP", "BIT", "VARDECIMAL", "BIT", "INT") + .go(); + } + + @Test + public void testModeOfLiteral() throws Exception { + String sql = + "SELECT modeOf(1) c1," + + "modeOf('a') c2," + + "modeOf(cast(null as int)) c3," + + "modeOf(case when true then null else 'a' end) c4," + + "modeOf(case when false then null else 'a' end) c5"; + + testBuilder() + .sqlQuery(sql) + .unOrdered() + .baselineColumns("c1", "c2", "c3", "c4", "c5") + .baselineValues("NOT NULL", "NOT NULL", "NULLABLE", "NULLABLE", "NULLABLE") + .go(); + } + + @Test + public void testCompareTypeLiteral() throws Exception { + String sql = + "SELECT compareType(1, 2) c1," + + "compareType('a', 1) c2," + + "compareType(1, 'a') c3," + + "compareType(a, '01:00:20.123') c4," + + "compareType(3, t.a) c5," + + "compareType(t.a, 3) c6\n" + + "from (values(1)) t(a)"; + + testBuilder() + .sqlQuery(sql) + .unOrdered() + .baselineColumns("c1", "c2", "c3", "c4", "c5", "c6") + .baselineValues(0, 1, -1, -1, 0, 0) + .go(); + } } diff --git a/exec/vector/src/main/codegen/templates/BasicTypeHelper.java b/exec/vector/src/main/codegen/templates/BasicTypeHelper.java index 430a41b..383e195 100644 --- a/exec/vector/src/main/codegen/templates/BasicTypeHelper.java +++ b/exec/vector/src/main/codegen/templates/BasicTypeHelper.java @@ -207,7 +207,35 @@ public class BasicTypeHelper { throw new UnsupportedOperationException(buildErrorMessage("get writer implementation", type, mode)); } - public static Class<?> getHolderReaderImpl( MinorType type, DataMode mode){ + /** + * Creates and returns {@link FieldReader} instance for specified {@code MajorType type} using specisied {@code ValueHolder} + * + * @param type type of resulting {@link FieldReader} instance + * @param holder value holder for {@link FieldReader} creation + * @return {@link FieldReader} instance + */ + public static FieldReader getHolderReaderImpl(MajorType type, ValueHolder holder) { + switch (type.getMinorType()) { + <#list vv.types as type> + <#list type.minor as minor> + case ${minor.class?upper_case}: + switch (type.getMode()) { + case REQUIRED: + return new ${minor.class}HolderReaderImpl((${minor.class}Holder) holder); + case OPTIONAL: + return new Nullable${minor.class}HolderReaderImpl((Nullable${minor.class}Holder) holder); + case REPEATED: + return new Repeated${minor.class}HolderReaderImpl((Repeated${minor.class}Holder) holder); + } + </#list> + </#list> + case NULL: + return new UntypedHolderReaderImpl((UntypedNullHolder) holder); + } + throw new UnsupportedOperationException(buildErrorMessage("get holder reader implementation", type.getMinorType(), type.getMode())); + } + + public static Class<?> getHolderReaderImpl(MinorType type, DataMode mode) { switch (type) { <#list vv.types as type> <#list type.minor as minor>
