DRILL-320: Rework math functions to use code generated using free marker. Contains implementation for basic functions: +, -, *, /
Signed-off-by: Jacques Nadeau <jacq...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ed9dec7c Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ed9dec7c Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ed9dec7c Branch: refs/heads/master Commit: ed9dec7c3f289e4f68466b989a500523f3d18b1e Parents: 8b90cae Author: Mehant Baid <meha...@gmail.com> Authored: Fri Feb 7 22:54:06 2014 -0800 Committer: Jacques Nadeau <jacq...@apache.org> Committed: Mon Mar 3 23:22:17 2014 -0800 ---------------------------------------------------------------------- exec/java-exec/src/main/codegen/config.fmpp | 3 +- .../src/main/codegen/data/MathFunctionTypes.tdd | 72 +++++++++++++++ .../templates/MathFunctionTemplates.java | 70 ++++++++++++++ .../drill/exec/expr/fn/impl/MathFunctions.java | 59 ------------ .../drill/exec/fn/impl/TestMathFunctions.java | 96 ++++++++++++++++++++ .../functions/simple_math_functions.json | 34 +++++++ 6 files changed, 274 insertions(+), 60 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed9dec7c/exec/java-exec/src/main/codegen/config.fmpp ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/config.fmpp b/exec/java-exec/src/main/codegen/config.fmpp index cd2b2cc..8f1060a 100644 --- a/exec/java-exec/src/main/codegen/config.fmpp +++ b/exec/java-exec/src/main/codegen/config.fmpp @@ -17,7 +17,8 @@ data: { vv: tdd(../data/ValueVectorTypes.tdd), compare: tdd(../data/CompareTypes.tdd), - cast: tdd(../data/Casts.tdd) + cast: tdd(../data/Casts.tdd), + MathFunctionTypes: tdd(../data/MathFunctionTypes.tdd) } freemarkerLinks: { includes: includes/ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed9dec7c/exec/java-exec/src/main/codegen/data/MathFunctionTypes.tdd ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/data/MathFunctionTypes.tdd b/exec/java-exec/src/main/codegen/data/MathFunctionTypes.tdd new file mode 100644 index 0000000..1724373 --- /dev/null +++ b/exec/java-exec/src/main/codegen/data/MathFunctionTypes.tdd @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:# www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{ + mathFunctionTypes: [ + {className: "Add", funcName: "add", op: "+", types: [ + {input1: "Int", input2: "Int", outputType: "Int", castType: "int"}, + {input1: "BigInt", input2: "BigInt", outputType: "BigInt", castType: "long"}, + {input1: "Float4", input2: "Float4", outputType: "Float4", castType: "float"}, + {input1: "Float8", input2: "Float8", outputType: "Float8", castType: "double"}, + {input1: "SmallInt", input2: "SmallInt", outputType: "SmallInt", castType: "short"}, + {input1: "TinyInt", input2: "TinyInt", outputType: "TinyInt", castType: "byte"}, + {input1: "UInt1", input2: "UInt1", outputType: "UInt1", castType: "byte"}, + {input1: "UInt2", input2: "UInt2", outputType: "UInt2", castType: "char"}, + {input1: "UInt4", input2: "UInt4", outputType: "UInt4", castType: "int"}, + {input1: "UInt8", input2: "UInt8", outputType: "UInt8", castType: "long"} + ] + }, + {className: "Subtract", funcName: "subtract", op: "-", types: [ + {input1: "Int", input2: "Int", outputType: "Int", castType: "int"}, + {input1: "BigInt", input2: "BigInt", outputType: "BigInt", castType: "long"}, + {input1: "Float4", input2: "Float4", outputType: "Float4", castType: "float"}, + {input1: "Float8", input2: "Float8", outputType: "Float8", castType: "double"}, + {input1: "SmallInt", input2: "SmallInt", outputType: "SmallInt", castType: "short"}, + {input1: "TinyInt", input2: "TinyInt", outputType: "TinyInt", castType: "byte"}, + {input1: "UInt1", input2: "UInt1", outputType: "UInt1", castType: "byte"}, + {input1: "UInt2", input2: "UInt2", outputType: "UInt2", castType: "char"}, + {input1: "UInt4", input2: "UInt4", outputType: "UInt4", castType: "int"}, + {input1: "UInt8", input2: "UInt8", outputType: "UInt8", castType: "long"} + ] + }, + {className: "Multiply", funcName: "multiply", op: "*", types: [ + {input1: "Int", input2: "Int", outputType: "Int", castType: "int"}, + {input1: "BigInt", input2: "BigInt", outputType: "BigInt", castType: "long"}, + {input1: "Float4", input2: "Float4", outputType: "Float4", castType: "float"}, + {input1: "Float8", input2: "Float8", outputType: "Float8", castType: "double"}, + {input1: "SmallInt", input2: "SmallInt", outputType: "SmallInt", castType: "short"}, + {input1: "TinyInt", input2: "TinyInt", outputType: "TinyInt", castType: "byte"}, + {input1: "UInt1", input2: "UInt1", outputType: "UInt1", castType: "byte"}, + {input1: "UInt2", input2: "UInt2", outputType: "UInt2", castType: "char"}, + {input1: "UInt4", input2: "UInt4", outputType: "UInt4", castType: "int"}, + {input1: "UInt8", input2: "UInt8", outputType: "UInt8", castType: "long"} + ] + }, + {className: "Divide", funcName: "divide", op: "/", types: [ + {input1: "Int", input2: "Int", outputType: "Int", castType: "int"}, + {input1: "BigInt", input2: "BigInt", outputType: "BigInt", castType: "long"}, + {input1: "Float4", input2: "Float4", outputType: "Float4", castType: "float"}, + {input1: "Float8", input2: "Float8", outputType: "Float8", castType: "double"}, + {input1: "SmallInt", input2: "SmallInt", outputType: "SmallInt", castType: "short"}, + {input1: "TinyInt", input2: "TinyInt", outputType: "TinyInt", castType: "byte"}, + {input1: "UInt1", input2: "UInt1", outputType: "UInt1", castType: "byte"}, + {input1: "UInt2", input2: "UInt2", outputType: "UInt2", castType: "char"}, + {input1: "UInt4", input2: "UInt4", outputType: "UInt4", castType: "int"}, + {input1: "UInt8", input2: "UInt8", outputType: "UInt8", castType: "long"} + ] + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed9dec7c/exec/java-exec/src/main/codegen/templates/MathFunctionTemplates.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/MathFunctionTemplates.java b/exec/java-exec/src/main/codegen/templates/MathFunctionTemplates.java new file mode 100644 index 0000000..9ec2178 --- /dev/null +++ b/exec/java-exec/src/main/codegen/templates/MathFunctionTemplates.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +<@pp.dropOutputFile /> + + + +<#list MathFunctionTypes.mathFunctionTypes as inputType> +<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/${inputType.className}Functions.java" /> + +<#include "/@includes/license.ftl" /> + +<#-- A utility class that is used to generate java code for add function --> + +/* + * This class is automatically generated from AddTypes.tdd using FreeMarker. + */ + + +package org.apache.drill.exec.expr.fn.impl; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.annotations.Workspace; +import org.apache.drill.exec.expr.holders.*; +import org.apache.drill.exec.record.RecordBatch; + +@SuppressWarnings("unused") + +public class ${inputType.className}Functions { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${inputType.className}Functions.class); + +<#list inputType.types as type> + + @FunctionTemplate(name = "${inputType.funcName}", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) + public static class ${type.input1}${type.input2}${inputType.className} implements DrillSimpleFunc { + + @Param ${type.input1}Holder in1; + @Param ${type.input2}Holder in2; + @Output ${type.outputType}Holder out; + + public void setup(RecordBatch b) { + } + + public void eval() { + out.value = (${type.castType}) (in1.value ${inputType.op} in2.value); + } + } +</#list> +} +</#list> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed9dec7c/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java index ea251c3..f4e2060 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java @@ -34,65 +34,6 @@ public class MathFunctions{ private MathFunctions(){} - @FunctionTemplate(name = "add", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) - public static class Add1 implements DrillSimpleFunc{ - - @Param IntHolder left; - @Param IntHolder right; - @Output IntHolder out; - - public void setup(RecordBatch b){} - - public void eval(){ - out.value = left.value + right.value; - } - - } - - @FunctionTemplate(name = "add", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) - public static class LongAdd1 implements DrillSimpleFunc{ - - @Param BigIntHolder left; - @Param BigIntHolder right; - @Output BigIntHolder out; - - public void setup(RecordBatch b){} - - public void eval(){ - out.value = left.value + right.value; - } - - } - - @FunctionTemplate(name = "add", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) - public static class Float4Add implements DrillSimpleFunc{ - - @Param Float4Holder left; - @Param Float4Holder right; - @Output Float4Holder out; - - public void setup(RecordBatch b){} - - public void eval(){ - out.value = left.value + right.value; - } - } - - - @FunctionTemplate(name = "add", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) - public static class Float8Add implements DrillSimpleFunc{ - - @Param Float8Holder left; - @Param Float8Holder right; - @Output Float8Holder out; - - public void setup(RecordBatch b){} - - public void eval(){ - out.value = left.value + right.value; - } - } - @FunctionTemplate(name = "negative", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) public static class Negative implements DrillSimpleFunc{ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed9dec7c/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java new file mode 100644 index 0000000..cc90f13 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.drill.exec.fn.impl; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; + +import mockit.Injectable; +import mockit.NonStrictExpectations; + +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.expression.ExpressionPosition; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.util.FileUtils; +import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; +import org.apache.drill.exec.memory.BufferAllocator; +import org.apache.drill.exec.memory.TopLevelAllocator; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.physical.PhysicalPlan; +import org.apache.drill.exec.physical.base.FragmentRoot; +import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; +import org.apache.drill.exec.physical.impl.ImplCreator; +import org.apache.drill.exec.physical.impl.SimpleRootExec; +import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.proto.BitControl; +import org.apache.drill.exec.proto.CoordinationProtos; +import org.apache.drill.exec.proto.ExecProtos.FragmentHandle; +import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.vector.BigIntVector; +import org.apache.drill.exec.vector.Float8Vector; + +import org.junit.Test; + +import com.google.common.base.Charsets; +import com.google.common.io.Files; +import com.codahale.metrics.MetricRegistry; + + +public class TestMathFunctions { + + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestMathFunctions.class); + DrillConfig c = DrillConfig.create(); + + + @Test + public void testBasicMathFunctions(@Injectable final DrillbitContext bitContext, @Injectable UserClientConnection connection) throws Throwable + { + + new NonStrictExpectations(){{ + bitContext.getMetrics(); result = new MetricRegistry(); + bitContext.getAllocator(); result = new TopLevelAllocator(); + bitContext.getConfig(); result = c; + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + }}; + + PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/simple_math_functions.json"), Charsets.UTF_8)); + FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); + FragmentContext context = new FragmentContext(bitContext, BitControl.PlanFragment.getDefaultInstance(), connection, registry); + SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next())); + + while(exec.next()) { + BigIntVector intMulVector = exec.getValueVectorById(new SchemaPath("INTMUL", ExpressionPosition.UNKNOWN), BigIntVector.class); + Float8Vector floatMulVector = exec.getValueVectorById(new SchemaPath("FLOATMUL", ExpressionPosition.UNKNOWN), Float8Vector.class); + BigIntVector intAddVector = exec.getValueVectorById(new SchemaPath("INTADD", ExpressionPosition.UNKNOWN), BigIntVector.class); + Float8Vector floatAddVector = exec.getValueVectorById(new SchemaPath("FLOATADD", ExpressionPosition.UNKNOWN), Float8Vector.class); + assertEquals(exec.getRecordCount(), 1); + assertEquals(intMulVector.getAccessor().get(0), 2); + assertEquals(floatMulVector.getAccessor().get(0), (1.1 * 2.2), 0); + assertEquals(intAddVector.getAccessor().get(0), 3); + assertEquals(floatAddVector.getAccessor().get(0), (1.1 + 2.2), 0); + } + + if(context.getFailureCause() != null){ + throw context.getFailureCause(); + } + assertTrue(!context.isFailed()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed9dec7c/exec/java-exec/src/test/resources/functions/simple_math_functions.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/functions/simple_math_functions.json b/exec/java-exec/src/test/resources/functions/simple_math_functions.json new file mode 100644 index 0000000..28ac5b2 --- /dev/null +++ b/exec/java-exec/src/test/resources/functions/simple_math_functions.json @@ -0,0 +1,34 @@ +{ + head : { + version : 1, + generator : { + type : "optiq", + info : "na" + }, + type : "APACHE_DRILL_PHYSICAL" + }, + graph:[ + { + @id:1, + pop:"mock-sub-scan", + url: "http://apache.org";, + entries:[ + {records: 1, types: [ + {name: "blue", type: "BIGINT", mode: "REQUIRED"} + ]} + ] + }, { + pop : "project", + @id : 2, + exprs : [ + { ref : "output.INTMUL", expr : " 1 * 2 "}, + { ref : "output.FLOATMUL", expr : " 1.1 * 2.2 "}, + { ref : "output.INTADD", expr : " 1 + 2 " }, + { ref : "output.FLOATADD", expr : " 1.1 + 2.2 " } ], + child : 1 + }, { + pop : "screen", + @id : 3, + child : 2 + } ] +} \ No newline at end of file
