Repository: incubator-drill Updated Branches: refs/heads/master 2ce675679 -> 30ce7d710
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java index 0de64b4..679001c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java @@ -30,7 +30,6 @@ import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.ErrorCollector; import org.apache.drill.common.expression.ErrorCollectorImpl; import org.apache.drill.common.expression.ExpressionPosition; -import org.apache.drill.common.expression.FunctionRegistry; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.expression.parser.ExprLexer; @@ -100,7 +99,6 @@ public class ExpressionTest { ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr)); CommonTokenStream tokens = new CommonTokenStream(lexer); ExprParser parser = new ExprParser(tokens); - parser.setRegistry(new FunctionRegistry(DrillConfig.create())); parse_return ret = parser.parse(); return ret.e; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java index 7ec7243..738d7f4 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java @@ -17,10 +17,6 @@ */ package org.apache.drill.exec.fn.impl; -import org.apache.drill.common.expression.ArgumentValidators; -import org.apache.drill.common.expression.CallProvider; -import org.apache.drill.common.expression.FunctionDefinition; -import org.apache.drill.common.expression.OutputTypeDeterminer; import org.apache.drill.exec.expr.DrillSimpleFunc; import org.apache.drill.exec.expr.annotations.FunctionTemplate; import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; @@ -35,25 +31,9 @@ import java.util.Random; public class GeneratorFunctions { public static final Random random = new Random(1234L); - public static final FunctionDefinition RANDOM_BIG_INT = FunctionDefinition.simpleRandom("randomBigInt", new ArgumentValidators.NumericTypeAllowed(1,2, true), - OutputTypeDeterminer.FIXED_BIGINT, "randomBigInt"); - public static final FunctionDefinition RANDOM_FLOAT8 = FunctionDefinition.simpleRandom("randomFloat8", new ArgumentValidators.NumericTypeAllowed(1,2, true), - OutputTypeDeterminer.FIXED_FLOAT8, "randomFloat8"); - public static final FunctionDefinition INCREASING_BIGINT = FunctionDefinition.simpleRandom("increasingBigInt", new ArgumentValidators.NumericTypeAllowed(1, true), - OutputTypeDeterminer.FIXED_BIGINT, "increasingBigInt"); - - public static class Provider implements CallProvider { - - @Override - public FunctionDefinition[] getFunctionDefintions() { - return new FunctionDefinition[] { RANDOM_BIG_INT, - RANDOM_FLOAT8, - INCREASING_BIGINT }; - } - - } - @FunctionTemplate(name = "increasingBigInt", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) + @FunctionTemplate(name = "increasingBigInt", isRandom = true, + scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) public static class IncreasingBigInt implements DrillSimpleFunc { @Param BigIntHolder start; @@ -69,7 +49,8 @@ public class GeneratorFunctions { } } - @FunctionTemplate(name = "randomBigInt", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) + @FunctionTemplate(name = "randomBigInt", isRandom = true, + scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) public static class RandomBigIntGauss implements DrillSimpleFunc { @Param BigIntHolder range; @@ -83,7 +64,8 @@ public class GeneratorFunctions { } } - @FunctionTemplate(name = "randomBigInt", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) + @FunctionTemplate(name = "randomBigInt", isRandom = true, + scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) public static class RandomBigInt implements DrillSimpleFunc { @Param BigIntHolder min; @@ -98,7 +80,8 @@ public class GeneratorFunctions { } } - @FunctionTemplate(name = "randomFloat8", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) + @FunctionTemplate(name = "randomFloat8", isRandom = true, + scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) public static class RandomFloat8Gauss implements DrillSimpleFunc { @Param BigIntHolder range; @@ -113,7 +96,8 @@ public class GeneratorFunctions { } } - @FunctionTemplate(name = "randomFloat8", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) + @FunctionTemplate(name = "randomFloat8", isRandom = true, + scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) public static class RandomFloat8 implements DrillSimpleFunc { @Param BigIntHolder min; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java new file mode 100644 index 0000000..eb9bd0e --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java @@ -0,0 +1,213 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.physical.impl; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import mockit.Injectable; +import mockit.NonStrictExpectations; + +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; +import org.apache.drill.exec.memory.TopLevelAllocator; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.physical.PhysicalPlan; +import org.apache.drill.exec.physical.base.FragmentRoot; +import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.proto.BitControl.PlanFragment; +import org.apache.drill.exec.proto.CoordinationProtos; +import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.vector.Float4Vector; +import org.apache.drill.exec.vector.NullableBigIntVector; +import org.apache.drill.exec.vector.NullableBitVector; +import org.apache.drill.exec.vector.NullableFloat8Vector; +import org.apache.drill.exec.vector.NullableIntVector; +import org.apache.drill.exec.vector.NullableVar16CharVector; +import org.apache.drill.exec.vector.ValueVector; +import org.apache.drill.exec.vector.Var16CharVector; +import org.junit.Test; + +import com.codahale.metrics.MetricRegistry; +import com.google.common.base.Charsets; +import com.google.common.io.Resources; + +import java.util.Iterator; + +public class TestHiveUDFs { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestHiveUDFs.class); + + DrillConfig c = DrillConfig.create(); + PhysicalPlanReader reader; + FunctionImplementationRegistry registry; + FragmentContext context; + + private void setup(final DrillbitContext bitContext, UserClientConnection connection) throws Throwable { + if(reader == null) + reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + if(registry == null) + registry = new FunctionImplementationRegistry(c); + if(context == null) + context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); + } + + @Test + public void testGenericUDF(@Injectable final DrillbitContext bitContext, + @Injectable UserClientConnection connection) throws Throwable { + new NonStrictExpectations(){{ + bitContext.getMetrics(); result = new MetricRegistry(); + bitContext.getAllocator(); result = new TopLevelAllocator(); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + }}; + + String planString = Resources.toString(Resources.getResource("functions/hive/GenericUDF.json"), Charsets.UTF_8); + + setup(bitContext, connection); + PhysicalPlan plan = reader.readPhysicalPlan(planString); + SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next())); + + int numRecords = 0; + + while(exec.next()){ + // Output columns and types + // 1. str1 : Var16Char + // 2. upperStr1 : NullableVar16Char + // 3. unix_timestamp : NullableBigInt + // 4. concat : NullableVarChar + // 5. flt1 : Float4 + // 6. format_number : NullableFloat8 + // 7. nullableStr1 : NullableVar16Char + // 8. upperNullableStr1 : NullableVar16Char + Iterator<ValueVector> vv = exec.iterator(); + Var16CharVector str1V = (Var16CharVector) vv.next(); + NullableVar16CharVector upperStr1V = (NullableVar16CharVector) vv.next(); + NullableBigIntVector unix_timestampV = (NullableBigIntVector) vv.next(); + NullableVar16CharVector concatV = (NullableVar16CharVector) vv.next(); + Float4Vector flt1V = (Float4Vector) vv.next(); + NullableVar16CharVector format_numberV = (NullableVar16CharVector) vv.next(); + + /* DRILL-425 + NullableVar16CharVector nullableStr1V = ((NullableVar16CharVector) vv.next()); + NullableVar16CharVector upperNullableStr1V = ((NullableVar16CharVector) vv.next()); */ + + for(int i=0; i<exec.getRecordCount(); i++) { + + + String in = new String(str1V.getAccessor().get(i)); + String upper = new String(upperStr1V.getAccessor().get(i)); + assertTrue(in.toUpperCase().equals(upper)); + + long unix_timestamp = unix_timestampV.getAccessor().get(i); + + String concat = new String(concatV.getAccessor().get(i)); + assertTrue(concat.equals(in+"-"+in)); + + float flt1 = flt1V.getAccessor().get(i); + String format_number = new String(format_numberV.getAccessor().get(i)); + + + String nullableStr1 = null; + /* DRILL-425 + if (!nullableStr1V.getAccessor().isNull(i)) + nullableStr1 = new String(nullableStr1V.getAccessor().get(i));*/ + + String upperNullableStr1 = null; + /* DRILL-425 + if (!upperNullableStr1V.getAccessor().isNull(i)) + upperNullableStr1 = new String(upperNullableStr1V.getAccessor().get(i)); */ + + assertEquals(nullableStr1 != null, upperNullableStr1 != null); + if (nullableStr1 != null) + assertEquals(nullableStr1.toUpperCase(), upperNullableStr1); + + System.out.println(in + ", " + upper + ", " + unix_timestamp + ", " + concat + ", " + + flt1 + ", " + format_number + ", " + nullableStr1 + ", " + upperNullableStr1); + + numRecords++; + } + } + + System.out.println("Processed " + numRecords + " records"); + + if(context.getFailureCause() != null){ + throw context.getFailureCause(); + } + + assertTrue(!context.isFailed()); + } + + @Test + public void testUDF(@Injectable final DrillbitContext bitContext, + @Injectable UserClientConnection connection) throws Throwable { + new NonStrictExpectations(){{ + bitContext.getMetrics(); result = new MetricRegistry(); + bitContext.getAllocator(); result = new TopLevelAllocator(); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + }}; + + String planString = Resources.toString(Resources.getResource("functions/hive/UDF.json"), Charsets.UTF_8); + + setup(bitContext, connection); + PhysicalPlan plan = reader.readPhysicalPlan(planString); + SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next())); + + int numRecords = 0; + while(exec.next()){ + // Output columns and types + // 1. str1 : Var16Char + // 2. str1Length : Int + // 3. str1Ascii : Int + // 4. flt1 : Float4 + // 5. pow : Float8 + Iterator<ValueVector> vv = exec.iterator(); + Var16CharVector str1V = (Var16CharVector) vv.next(); + NullableIntVector str1LengthV = (NullableIntVector) vv.next(); + NullableIntVector str1AsciiV = (NullableIntVector) vv.next(); + Float4Vector flt1V = (Float4Vector) vv.next(); + NullableFloat8Vector powV = (NullableFloat8Vector) vv.next(); + + for(int i=0; i<exec.getRecordCount(); i++) { + + String str1 = new String(str1V.getAccessor().get(i)); + int str1Length = str1LengthV.getAccessor().get(i); + assertTrue(str1.length() == str1Length); + + int str1Ascii = str1AsciiV.getAccessor().get(i); + + float flt1 = flt1V.getAccessor().get(i); + + double pow = 0; + if (!powV.getAccessor().isNull(i)) { + pow = powV.getAccessor().get(i); + assertTrue(Math.pow(flt1, 2.0) == pow); + } + + System.out.println(str1 + ", " + str1Length + ", " + str1Ascii + ", " + flt1 + ", " + pow); + numRecords++; + } + } + + System.out.println("Processed " + numRecords + " records"); + + if(context.getFailureCause() != null){ + throw context.getFailureCause(); + } + + assertTrue(!context.isFailed()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java index 5bfe369..767b75e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java @@ -143,15 +143,12 @@ public class TestSimpleFunctions { args.add(arg); String[] registeredNames = { "hash" }; FunctionCall call = new FunctionCall( - FunctionDefinition.simple("hash", - new BasicArgumentValidator(new Arg[]{ new Arg(expectedArg)}), - new OutputTypeDeterminer.FixedType(expectedOut), - registeredNames), + "hash", args, ExpressionPosition.UNKNOWN ); FunctionResolver resolver = FunctionResolverFactory.getResolver(call); - DrillFuncHolder matchedFuncHolder = resolver.getBestMatch(registry.getMethods().get(call.getDefinition().getName()), call); + DrillFuncHolder matchedFuncHolder = resolver.getBestMatch(registry.getDrillRegistry().getMethods().get(call.getName()), call); assertEquals( expectedBestInputMode, matchedFuncHolder.getParmMajorType(0).getMode()); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java index b3b7536..95bee87 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java @@ -21,22 +21,17 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.util.List; - import mockit.Injectable; import mockit.NonStrictExpectations; import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.expression.ArgumentValidator; import org.apache.drill.common.expression.ErrorCollector; import org.apache.drill.common.expression.ErrorCollectorImpl; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.FieldReference; import org.apache.drill.common.expression.FunctionCall; -import org.apache.drill.common.expression.FunctionDefinition; import org.apache.drill.common.expression.IfExpression; import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.expression.OutputTypeDeterminer; import org.apache.drill.common.expression.ValueExpressions; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; @@ -207,25 +202,11 @@ public class ExpressionTreeMaterializerTest { }; - LogicalExpression functionCallExpr = new FunctionCall(FunctionDefinition.simple("testFunc", - new ArgumentValidator() { - @Override - public void validateArguments(ExpressionPosition expr, List<LogicalExpression> expressions, - ErrorCollector errors) { - errors.addGeneralError(expr, "Error!"); - } - - @Override - public String[] getArgumentNamesByPosition() { - return new String[0]; - } - }, OutputTypeDeterminer.FIXED_BIT), ImmutableList.of((LogicalExpression) // - new FieldReference("test", ExpressionPosition.UNKNOWN) ), ExpressionPosition.UNKNOWN); + LogicalExpression functionCallExpr = new FunctionCall("testFunc", + ImmutableList.of((LogicalExpression) new FieldReference("test", ExpressionPosition.UNKNOWN) ), + ExpressionPosition.UNKNOWN); LogicalExpression newExpr = ExpressionTreeMaterializer.materialize(functionCallExpr, batch, ec, registry); - assertTrue(newExpr instanceof FunctionCall); - FunctionCall funcExpr = (FunctionCall) newExpr; - assertEquals(1, funcExpr.args.size()); - assertEquals(bigIntType, funcExpr.args.get(0).getMajorType()); + assertEquals(newExpr, null); assertEquals(1, ec.getErrorCount()); System.out.println(ec.toErrorString()); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/test/resources/functions/hive/GenericUDF.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/functions/hive/GenericUDF.json b/exec/java-exec/src/test/resources/functions/hive/GenericUDF.json new file mode 100644 index 0000000..7b82570 --- /dev/null +++ b/exec/java-exec/src/test/resources/functions/hive/GenericUDF.json @@ -0,0 +1,45 @@ +{ + head:{ + type:"APACHE_DRILL_PHYSICAL", + version:"1", + generator:{ + type:"manual" + } + }, + graph:[ + { + @id:1, + pop:"mock-sub-scan", + url: "http://apache.org";, + entries:[ + {records: 100, types: [ + {name: "str1", type: "VAR16CHAR", mode: "REQUIRED"}, + {name: "str2", type: "VAR16CHAR", mode: "REQUIRED"}, + {name: "str3", type: "VAR16CHAR", mode: "REQUIRED"}, + /* DRILL-425 {name: "nullableStr1", type: "VAR16CHAR", mode: "OPTIONAL"}, */ + {name: "flt1", type: "FLOAT4", mode: "REQUIRED"} + ]} + ] + }, + { + @id:2, + child: 1, + pop:"project", + exprs: [ + { ref: "str1", expr: "str1" }, + { ref: "upperStr1", expr: "ucase(str1)" }, + { ref: "unix_timestamp", expr: "unix_timestamp()" }, + { ref: "concat", expr: "concat_ws('-', str2, str3)" }, + { ref: "flt1", expr: "flt1" }, + { ref: "format_number", expr: "format_number(cast(flt1 as float8), cast(2 as int))" } /* DRILL-425, + { ref: "nullableStr1", expr: "nullableStr1" }, + { ref: "upperNulableStr1", expr: "upper(nullableStr1)" } */ + ] + }, + { + @id: 3, + child: 2, + pop: "screen" + } + ] +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/test/resources/functions/hive/UDF.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/functions/hive/UDF.json b/exec/java-exec/src/test/resources/functions/hive/UDF.json new file mode 100644 index 0000000..cd4c7b1 --- /dev/null +++ b/exec/java-exec/src/test/resources/functions/hive/UDF.json @@ -0,0 +1,39 @@ +{ + head:{ + type:"APACHE_DRILL_PHYSICAL", + version:"1", + generator:{ + type:"manual" + } + }, + graph:[ + { + @id:1, + pop:"mock-sub-scan", + url: "http://apache.org";, + entries:[ + {records: 100, types: [ + {name: "str1", type: "VAR16CHAR", mode: "REQUIRED"}, + {name: "flt1", type: "FLOAT4", mode: "REQUIRED"} + ]} + ] + }, + { + @id:2, + child: 1, + pop:"project", + exprs: [ + { ref: "str1", expr: "str1" }, + { ref: "str1Length", expr: "length(str1)" }, + { ref: "str1Ascii", expr: "ascii(str1)" }, + { ref: "flt1", expr: "flt1" }, + { ref: "pow", expr: "pow(cast(flt1 as float8), 2.0)" } + ] + }, + { + @id: 3, + child: 2, + pop: "screen" + } + ] +}
