DRILL-1383: Support interpreted execution for Drill expression tree. Fix OOM while running InterpreterBuilder
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/9bc71fc5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/9bc71fc5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/9bc71fc5 Branch: refs/heads/smp-merge-230914 Commit: 9bc71fc54b97b52ac5c7335247b6ca7887045fd2 Parents: 8b8d787 Author: Jinfeng Ni <j...@maprtech.com> Authored: Thu Sep 4 20:40:50 2014 -0700 Committer: Steven Phillips <sphill...@maprtech.com> Committed: Tue Sep 23 22:23:02 2014 -0700 ---------------------------------------------------------------------- exec/interpreter/pom.xml | 109 ++++++ .../exec/expr/ExpressionInterpreterTest.java | 201 +++++++++++ .../src/main/codegen/templates/TypeHelper.java | 217 +++++++++++- .../drill/exec/expr/DrillFuncHolderExpr.java | 10 + .../exec/expr/fn/DrillBooleanOPHolder.java | 2 +- .../drill/exec/expr/fn/DrillFuncHolder.java | 25 ++ .../exec/expr/fn/DrillSimpleFuncHolder.java | 27 +- .../drill/exec/expr/fn/FunctionConverter.java | 5 +- .../fn/interpreter/DrillFuncInterpreter.java | 26 ++ .../interpreter/DrillSimpleFuncInterpreter.java | 30 ++ .../expr/fn/interpreter/InterpreterBuilder.java | 69 ++++ .../fn/interpreter/InterpreterEvaluator.java | 334 +++++++++++++++++++ .../fn/interpreter/InterpreterGenerator.java | 143 ++++++++ .../drill/exec/vector/ValueHolderHelper.java | 52 +++ exec/pom.xml | 1 + 15 files changed, 1234 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9bc71fc5/exec/interpreter/pom.xml ---------------------------------------------------------------------- diff --git a/exec/interpreter/pom.xml b/exec/interpreter/pom.xml new file mode 100644 index 0000000..0773a71 --- /dev/null +++ b/exec/interpreter/pom.xml @@ -0,0 +1,109 @@ +<?xml version="1.0"?> +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.drill.exec</groupId> + <artifactId>exec-parent</artifactId> + <version>0.6.0-incubating-SNAPSHOT</version> + </parent> + <artifactId>drill-interpreter</artifactId> + <name>exec/Drill expression interpreter</name> + + <dependencies> + <dependency> + <groupId>org.apache.drill</groupId> + <artifactId>drill-common</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.drill.exec</groupId> + <artifactId>drill-java-exec</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.drill</groupId> + <artifactId>drill-common</artifactId> + <classifier>tests</classifier> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.drill.exec</groupId> + <artifactId>drill-java-exec</artifactId> + <version>${project.version}</version> + <classifier>tests</classifier> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.codehaus.janino</groupId> + <artifactId>janino</artifactId> + <version>2.6.1</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <dependencies> + <dependency> + <groupId>org.apache.drill.exec</groupId> + <artifactId>drill-java-exec</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <executions> + <execution> + <id>generate-sources</id> + <phase>generate-sources</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>java</executable> + <arguments> + <argument>-Xmx1024m</argument> + <argument>-XX:MaxPermSize=256m</argument> + <argument>-classpath</argument> + <classpath/> + <argument>org.apache.drill.exec.expr.fn.interpreter.InterpreterBuilder</argument> + <argument>${project.build.directory}/generated-sources</argument> + </arguments> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.9.1</version> + <executions> + <execution> + <id>add-generated-source</id> + <phase>generate-sources</phase> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>${project.build.directory}/generated-sources</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9bc71fc5/exec/interpreter/src/test/java/org/apache/drill/exec/expr/ExpressionInterpreterTest.java ---------------------------------------------------------------------- diff --git a/exec/interpreter/src/test/java/org/apache/drill/exec/expr/ExpressionInterpreterTest.java b/exec/interpreter/src/test/java/org/apache/drill/exec/expr/ExpressionInterpreterTest.java new file mode 100644 index 0000000..a94ef94 --- /dev/null +++ b/exec/interpreter/src/test/java/org/apache/drill/exec/expr/ExpressionInterpreterTest.java @@ -0,0 +1,201 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr; + + +import com.google.common.collect.Lists; +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.CommonTokenStream; +import org.antlr.runtime.RecognitionException; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.ErrorCollector; +import org.apache.drill.common.expression.ErrorCollectorImpl; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.parser.ExprLexer; +import org.apache.drill.common.expression.parser.ExprParser; +import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.common.types.Types; +import org.apache.drill.common.util.DrillStringUtils; +import org.apache.drill.exec.expr.fn.interpreter.InterpreterEvaluator; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.pop.PopUnitTestBase; +import org.apache.drill.exec.proto.BitControl; +import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.record.RecordBatch; +import org.apache.drill.exec.server.Drillbit; +import org.apache.drill.exec.server.RemoteServiceSet; +import org.apache.drill.exec.store.mock.MockGroupScanPOP; +import org.apache.drill.exec.store.mock.MockScanBatchCreator; +import org.apache.drill.exec.store.mock.MockSubScanPOP; +import org.apache.drill.exec.vector.ValueVector; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class ExpressionInterpreterTest extends PopUnitTestBase { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionInterpreterTest.class); + + @Test + public void interpreterNullableStrExpr() throws Exception { + String[] colNames = {"col1"}; + TypeProtos.MajorType[] colTypes = {Types.optional(TypeProtos.MinorType.VARCHAR)}; + String expressionStr = "substr(col1, 1, 3)"; + String[] expectedFirstTwoValues = {"aaa", "null"}; + + doTest(expressionStr, colNames, colTypes, expectedFirstTwoValues); + } + + + @Test + public void interpreterNullableBooleanExpr() throws Exception { + String[] colNames = {"col1"}; + TypeProtos.MajorType[] colTypes = {Types.optional(TypeProtos.MinorType.VARCHAR)}; + String expressionStr = "col1 < 'abc' and col1 > 'abc'"; + String[] expectedFirstTwoValues = {"false", "null"}; + + doTest(expressionStr, colNames, colTypes, expectedFirstTwoValues); + } + + + @Test + public void interpreterNullableIntegerExpr() throws Exception { + String[] colNames = {"col1"}; + TypeProtos.MajorType[] colTypes = {Types.optional(TypeProtos.MinorType.INT)}; + String expressionStr = "col1 + 100 - 1 * 2 + 2"; + String[] expectedFirstTwoValues = {"-2147483548", "null"}; + + doTest(expressionStr, colNames, colTypes, expectedFirstTwoValues); + } + + @Test + public void interpreterCaseExpr() throws Exception { + String[] colNames = {"col1"}; + TypeProtos.MajorType[] colTypes = {Types.optional(TypeProtos.MinorType.VARCHAR)}; + String expressionStr = "case when substr(col1, 1, 3)='aaa' then 'ABC' else 'XYZ' end"; + String[] expectedFirstTwoValues = {"ABC", "XYZ"}; + + doTest(expressionStr, colNames, colTypes, expectedFirstTwoValues); + } + + + protected void doTest(String expressionStr, String[] colNames, TypeProtos.MajorType[] colTypes, String[] expectFirstTwoValues) throws Exception { + RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet(); + + Drillbit bit1 = new Drillbit(CONFIG, serviceSet); + + bit1.run(); + + // Create a mock scan batch as input for evaluation. + assert(colNames.length == colTypes.length); + + MockGroupScanPOP.MockColumn[] columns = new MockGroupScanPOP.MockColumn[colNames.length]; + + for (int i = 0; i < colNames.length; i++ ) { + columns[i] = new MockGroupScanPOP.MockColumn(colNames[i], colTypes[i].getMinorType(), colTypes[i].getMode(),0,0,0); + } + + MockGroupScanPOP.MockScanEntry entry = new MockGroupScanPOP.MockScanEntry(10, columns); + MockSubScanPOP scanPOP = new MockSubScanPOP("testTable", java.util.Collections.singletonList(entry)); + + RecordBatch batch = createMockScanBatch(bit1, scanPOP); + + batch.next(); + + ValueVector vv = evalExprWithInterpreter(expressionStr, batch, bit1); + + // Verify the first 2 values in the output of evaluation. + assert(expectFirstTwoValues.length == 2); + assertEquals(expectFirstTwoValues[0], getValueFromVector(vv, 0)); + assertEquals(expectFirstTwoValues[1], getValueFromVector(vv, 1)); + + showValueVectorContent(vv); + + vv.clear(); + batch.cleanup(); + batch.getContext().close(); + bit1.close(); + } + + + private RecordBatch createMockScanBatch(Drillbit bit, MockSubScanPOP scanPOP) { + List<RecordBatch> children = Lists.newArrayList(); + MockScanBatchCreator creator = new MockScanBatchCreator(); + + try { + FragmentContext context = new FragmentContext(bit.getContext(), BitControl.PlanFragment.getDefaultInstance(), null, bit.getContext().getFunctionImplementationRegistry()); + return creator.getBatch(context,scanPOP, children); + } catch (Exception ex) { + throw new DrillRuntimeException("Error when setup fragment context" + ex); + } + } + + private LogicalExpression parseExpr(String expr) throws RecognitionException { + ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr)); + CommonTokenStream tokens = new CommonTokenStream(lexer); + ExprParser parser = new ExprParser(tokens); + ExprParser.parse_return ret = parser.parse(); + return ret.e; + } + + private ValueVector evalExprWithInterpreter(String expression, RecordBatch batch, Drillbit bit) throws Exception { + LogicalExpression expr = parseExpr(expression); + ErrorCollector error = new ErrorCollectorImpl(); + LogicalExpression materializedExpr = ExpressionTreeMaterializer.materialize(expr, batch, error, bit.getContext().getFunctionImplementationRegistry()); + if (error.getErrorCount() != 0) { + logger.error("Failure while materializing expression [{}]. Errors: {}", expression, error); + assertEquals(0, error.getErrorCount()); + } + + final MaterializedField outputField = MaterializedField.create("outCol", materializedExpr.getMajorType()); + + ValueVector vector = TypeHelper.getNewVector(outputField, bit.getContext().getAllocator()); + + vector.allocateNewSafe(); + + InterpreterEvaluator.evaluate(batch, vector, materializedExpr); + + return vector; + } + + private void showValueVectorContent(ValueVector vw) { + for (int row = 0; row < vw.getAccessor().getValueCount(); row ++ ) { + Object o = vw.getAccessor().getObject(row); + String cellString; + if (o instanceof byte[]) { + cellString = DrillStringUtils.toBinaryString((byte[]) o); + } else { + cellString = DrillStringUtils.escapeNewLines(String.valueOf(o)); + } + System.out.printf(row + "th value: " + cellString + "\n"); + } + } + + private String getValueFromVector(ValueVector vw, int index) { + Object o = vw.getAccessor().getObject(index); + String cellString; + if (o instanceof byte[]) { + cellString = DrillStringUtils.toBinaryString((byte[]) o); + } else { + cellString = DrillStringUtils.escapeNewLines(String.valueOf(o)); + } + return cellString; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9bc71fc5/exec/java-exec/src/main/codegen/templates/TypeHelper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java b/exec/java-exec/src/main/codegen/templates/TypeHelper.java index 9bb2507..3c0d9d3 100644 --- a/exec/java-exec/src/main/codegen/templates/TypeHelper.java +++ b/exec/java-exec/src/main/codegen/templates/TypeHelper.java @@ -16,6 +16,8 @@ * limitations under the License. */ +import org.apache.drill.common.types.MajorType; + <@pp.dropOutputFile /> <@pp.changeOutputFile name="/org/apache/drill/exec/expr/TypeHelper.java" /> @@ -305,11 +307,33 @@ public class TypeHelper { case ${minor.class?upper_case} : <#if minor.class?starts_with("Var") || minor.class == "TimeStampTZ" || minor.class == "IntervalDay" || minor.class == "Interval" || minor.class?starts_with("Decimal28") || minor.class?starts_with("Decimal38")> - throw new UnsupportedOperationException(type.getMinorType() + " type is not supported."); + switch (type.getMode()) { + case REQUIRED: + holder = new ${minor.class}Holder(); + ((${minor.class}Vector) vector).getAccessor().get(index, (${minor.class}Holder)holder); + return holder; + case OPTIONAL: + holder = new Nullable${minor.class}Holder(); + ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index); + if (((Nullable${minor.class}Holder)holder).isSet == 1) { + ((Nullable${minor.class}Vector) vector).getAccessor().get(index, (Nullable${minor.class}Holder)holder); + } + return holder; + } <#else> - holder = new ${minor.class}Holder(); - ((${minor.class}Holder)holder).value = ((${minor.class}Vector) vector).getAccessor().get(index); - break; + switch (type.getMode()) { + case REQUIRED: + holder = new ${minor.class}Holder(); + ((${minor.class}Holder)holder).value = ((${minor.class}Vector) vector).getAccessor().get(index); + return holder; + case OPTIONAL: + holder = new Nullable${minor.class}Holder(); + ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index); + if (((Nullable${minor.class}Holder)holder).isSet == 1) { + ((Nullable${minor.class}Holder)holder).value = ((Nullable${minor.class}Vector) vector).getAccessor().get(index); + } + return holder; + } </#if> </#list> </#list> @@ -318,9 +342,10 @@ public class TypeHelper { ((ObjectHolder)holder).obj = ((ObjectVector) vector).getAccessor().getObject(index) ; break; default: - throw new UnsupportedOperationException(type.getMinorType() + " type is not supported."); + throw new UnsupportedOperationException(type + " type is not supported."); } - return holder; + + throw new UnsupportedOperationException(type + " type is not supported."); } public static void setValue(ValueVector vector, int index, ValueHolder holder) { @@ -330,15 +355,54 @@ public class TypeHelper { <#list vv.types as type> <#list type.minor as minor> case ${minor.class?upper_case} : - ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder); - break; + switch (type.getMode()) { + case REQUIRED: + ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder); + return; + case OPTIONAL: + if (((Nullable${minor.class}Holder) holder).isSet == 1) { + ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder); + } + return; + } </#list> </#list> case GENERIC_OBJECT: ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder); - break ; + return; default: - throw new UnsupportedOperationException(type.getMinorType() + " type is not supported."); + throw new UnsupportedOperationException(type.getMinorType() + " type is not supported."); + } + } + + public static boolean setValueSafe(ValueVector vector, int index, ValueHolder holder) { + MajorType type = vector.getField().getType(); + + switch(type.getMinorType()) { + <#list vv.types as type> + <#list type.minor as minor> + case ${minor.class?upper_case} : + switch (type.getMode()) { + case REQUIRED: + return ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder); + case OPTIONAL: + if (((Nullable${minor.class}Holder) holder).isSet == 1) { + if (! ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder) ) { + return false; + } + } else { + if (!((Nullable${minor.class}Vector) vector).getMutator().isSafe(index)) { + return false; + } + } + return true; + } + </#list> + </#list> + case GENERIC_OBJECT: + return ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder); + default: + throw new UnsupportedOperationException(type.getMinorType() + " type is not supported."); } } @@ -366,4 +430,137 @@ public class TypeHelper { return false; } + /** + * Create a ValueHolder of MajorType. + * @param type + * @return + */ + public static ValueHolder createValueHolder(MajorType type) { + ValueHolder holder; + switch(type.getMinorType()) { + <#list vv.types as type> + <#list type.minor as minor> + case ${minor.class?upper_case} : + + switch (type.getMode()) { + case REQUIRED: + return new ${minor.class}Holder(); + case OPTIONAL: + return new Nullable${minor.class}Holder(); + case REPEATED: + return new Repeated${minor.class}Holder(); + } + </#list> + </#list> + case GENERIC_OBJECT: + return new ObjectHolder(); + default: + throw new UnsupportedOperationException(type + " type is not supported for 'createValueHolder' method."); + } + } + + public static boolean isNull(ValueHolder holder) { + MajorType type = TypeHelper.getValueHolderType(holder); + + switch(type.getMinorType()) { + <#list vv.types as type> + <#list type.minor as minor> + case ${minor.class?upper_case} : + + switch (type.getMode()) { + case REQUIRED: + return true; + case OPTIONAL: + return ((Nullable${minor.class}Holder) holder).isSet == 0; + case REPEATED: + return true; + } + </#list> + </#list> + default: + throw new UnsupportedOperationException(type + " type is not supported for 'isNull' method."); + } + } + + public static ValueHolder deNullify(ValueHolder holder) { + MajorType type = TypeHelper.getValueHolderType(holder); + + switch(type.getMinorType()) { + <#list vv.types as type> + <#list type.minor as minor> + case ${minor.class?upper_case} : + + switch (type.getMode()) { + case REQUIRED: + return holder; + case OPTIONAL: + if( ((Nullable${minor.class}Holder) holder).isSet == 1) { + ${minor.class}Holder newHolder = new ${minor.class}Holder(); + + <#assign fields = minor.fields!type.fields /> + <#list fields as field> + newHolder.${field.name} = ((Nullable${minor.class}Holder) holder).${field.name}; + </#list> + + return newHolder; + } else { + throw new UnsupportedOperationException("You can not convert a null value into a non-null value!"); + } + case REPEATED: + return holder; + } + </#list> + </#list> + default: + throw new UnsupportedOperationException(type + " type is not supported for 'deNullify' method."); + } + } + + public static ValueHolder nullify(ValueHolder holder) { + MajorType type = TypeHelper.getValueHolderType(holder); + + switch(type.getMinorType()) { + <#list vv.types as type> + <#list type.minor as minor> + case ${minor.class?upper_case} : + switch (type.getMode()) { + case REQUIRED: + Nullable${minor.class}Holder newHolder = new Nullable${minor.class}Holder(); + newHolder.isSet = 1; + <#assign fields = minor.fields!type.fields /> + <#list fields as field> + newHolder.${field.name} = ((${minor.class}Holder) holder).${field.name}; + </#list> + return newHolder; + case OPTIONAL: + return holder; + case REPEATED: + throw new UnsupportedOperationException("You can not convert repeated type " + type + " to nullable type!"); + } + </#list> + </#list> + default: + throw new UnsupportedOperationException(type + " type is not supported for 'nullify' method."); + } + } + + public static MajorType getValueHolderType(ValueHolder holder) { + + if (0 == 1) { + return null; + } + <#list vv.types as type> + <#list type.minor as minor> + else if (holder instanceof ${minor.class}Holder) { + return ((${minor.class}Holder) holder).TYPE; + } else if (holder instanceof Nullable${minor.class}Holder) { + return ((Nullable${minor.class}Holder) holder).TYPE; + } + </#list> + </#list> + + throw new UnsupportedOperationException("ValueHolder is not supported for 'getValueHolderType' method."); + + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9bc71fc5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java index d1734d4..bc631b8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java @@ -27,10 +27,12 @@ import org.apache.drill.common.expression.fn.FuncHolder; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.exec.expr.fn.DrillComplexWriterFuncHolder; import org.apache.drill.exec.expr.fn.DrillFuncHolder; +import org.apache.drill.exec.expr.fn.interpreter.DrillSimpleFuncInterpreter; public class DrillFuncHolderExpr extends FunctionHolderExpression implements Iterable<LogicalExpression>{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFuncHolderExpr.class); private DrillFuncHolder holder; + private DrillSimpleFuncInterpreter interpreter; public DrillFuncHolderExpr(String nameUsed, DrillFuncHolder holder, List<LogicalExpression> args, ExpressionPosition pos) { super(nameUsed, pos, args); @@ -92,5 +94,13 @@ public class DrillFuncHolderExpr extends FunctionHolderExpression implements Ite return new DrillFuncHolderExpr(this.nameUsed, this.holder, args, this.getPosition()); } + public void setInterpreter(DrillSimpleFuncInterpreter interpreter) { + this.interpreter = interpreter; + } + + public DrillSimpleFuncInterpreter getInterpreter() { + return this.interpreter; + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9bc71fc5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillBooleanOPHolder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillBooleanOPHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillBooleanOPHolder.java index 9032d37..743598a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillBooleanOPHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillBooleanOPHolder.java @@ -30,7 +30,7 @@ public class DrillBooleanOPHolder extends DrillSimpleFuncHolder{ public DrillBooleanOPHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, Map<String, String> methods, List<String> imports) { - super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports, FunctionCostCategory.getDefault()); + super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports); } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9bc71fc5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java index 1852731..279c428 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java @@ -277,6 +277,14 @@ public abstract class DrillFuncHolder extends AbstractFuncHolder { + ", parameters=" + (parameters != null ? Arrays.asList(parameters).subList(0, Math.min(parameters.length, maxLen)) : null) + "]"; } + public WorkspaceReference[] getWorkspaceVars() { + return this.workspaceVars; + } + + public ValueReference[] getParameters() { + return this.parameters; + } + public static class ValueReference { MajorType type; String name; @@ -292,6 +300,14 @@ public abstract class DrillFuncHolder extends AbstractFuncHolder { this.name = name; } + public MajorType getType() { + return type; + } + + public String getName() { + return name; + } + public void setConstant(boolean isConstant) { this.isConstant = isConstant; } @@ -344,6 +360,15 @@ public abstract class DrillFuncHolder extends AbstractFuncHolder { public boolean isInject() { return inject; } + + public Class<?> getType() { + return type; + } + + public String getName() { + return name; + } + } public boolean checkPrecisionRange() { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9bc71fc5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java index 4731200..ec284a7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java @@ -37,24 +37,27 @@ import com.sun.codemodel.JExpr; import com.sun.codemodel.JExpression; import com.sun.codemodel.JMod; import com.sun.codemodel.JVar; +import org.apache.drill.exec.expr.fn.interpreter.DrillSimpleFuncInterpreter; +import org.apache.drill.exec.expr.fn.interpreter.InterpreterGenerator; -class DrillSimpleFuncHolder extends DrillFuncHolder{ +public class DrillSimpleFuncHolder extends DrillFuncHolder{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillSimpleFuncHolder.class); private final String setupBody; private final String evalBody; private final String resetBody; private final String cleanupBody; + private final String interpreterClassName; public DrillSimpleFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, Map<String, String> methods, List<String> imports) { - this(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports, FunctionCostCategory.getDefault()); + this(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports, FunctionCostCategory.getDefault(), null); } public DrillSimpleFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, - Map<String, String> methods, List<String> imports, FunctionCostCategory costCategory) { + Map<String, String> methods, List<String> imports, FunctionCostCategory costCategory, String interpreterClassName) { super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports, costCategory); setupBody = methods.get("setup"); evalBody = methods.get("eval"); @@ -62,6 +65,7 @@ class DrillSimpleFuncHolder extends DrillFuncHolder{ cleanupBody = methods.get("cleanup"); Preconditions.checkNotNull(evalBody); + this.interpreterClassName = interpreterClassName; } @Override @@ -69,8 +73,14 @@ class DrillSimpleFuncHolder extends DrillFuncHolder{ return false; } - @Override - public HoldingContainer renderEnd(ClassGenerator<?> g, HoldingContainer[] inputVariables, JVar[] workspaceJVars) { + public DrillSimpleFuncInterpreter createInterpreter() throws Exception { + Preconditions.checkArgument(this.interpreterClassName != null, "interpreterClassName should not be null!"); + + String className = InterpreterGenerator.PACKAGE_NAME + "." + this.interpreterClassName; + return (DrillSimpleFuncInterpreter) Class.forName(className).newInstance(); + } + + public HoldingContainer renderEnd(ClassGenerator<?> g, HoldingContainer[] inputVariables, JVar[] workspaceJVars){ //If the function's annotation specifies a parameter has to be constant expression, but the HoldingContainer //for the argument is not, then raise exception. for (int i =0; i < inputVariables.length; i++) { @@ -141,4 +151,11 @@ class DrillSimpleFuncHolder extends DrillFuncHolder{ return out; } + public String getSetupBody() { + return setupBody; + } + + public String getEvalBody() { + return evalBody; + } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9bc71fc5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java index 402a6fc..0127e6e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java @@ -31,6 +31,7 @@ import javax.inject.Inject; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.util.FileUtils; +import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.expr.DrillFunc; import org.apache.drill.exec.expr.annotations.FunctionTemplate; import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; @@ -39,6 +40,7 @@ import org.apache.drill.exec.expr.annotations.Param; import org.apache.drill.exec.expr.annotations.Workspace; import org.apache.drill.exec.expr.fn.DrillFuncHolder.ValueReference; import org.apache.drill.exec.expr.fn.DrillFuncHolder.WorkspaceReference; +import org.apache.drill.exec.expr.fn.interpreter.InterpreterGenerator; import org.apache.drill.exec.expr.holders.ValueHolder; import org.apache.drill.exec.vector.complex.reader.FieldReader; import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter; @@ -257,7 +259,8 @@ public class FunctionConverter { return new DrillSimpleFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), template.isRandom(), registeredNames, - ps, outputField, works, methods, imports, template.costCategory()); + ps, outputField, works, methods, imports, template.costCategory(), + clazz.getSimpleName() + InterpreterGenerator.INTERPRETER_CLASSNAME_POSTFIX); } case SC_BOOLEAN_OPERATOR: return new DrillBooleanOPHolder(template.scope(), template.nulls(), http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9bc71fc5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/DrillFuncInterpreter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/DrillFuncInterpreter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/DrillFuncInterpreter.java new file mode 100644 index 0000000..3a83542 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/DrillFuncInterpreter.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.drill.exec.expr.fn.interpreter; + +import org.apache.drill.exec.expr.holders.ValueHolder; +import org.apache.drill.exec.record.RecordBatch; + +public interface DrillFuncInterpreter { + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9bc71fc5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/DrillSimpleFuncInterpreter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/DrillSimpleFuncInterpreter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/DrillSimpleFuncInterpreter.java new file mode 100644 index 0000000..e3696f0 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/DrillSimpleFuncInterpreter.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.drill.exec.expr.fn.interpreter; + +import org.apache.drill.exec.expr.holders.ValueHolder; +import org.apache.drill.exec.record.RecordBatch; + +public interface DrillSimpleFuncInterpreter extends DrillFuncInterpreter { + + public void doSetup(ValueHolder[] args, RecordBatch incoming); + + public ValueHolder doEval(ValueHolder [] args) ; + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9bc71fc5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterBuilder.java new file mode 100644 index 0000000..3dac818 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterBuilder.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr.fn.interpreter; + +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.util.PathScanner; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.expr.DrillFunc; +import org.apache.drill.exec.expr.fn.DrillFuncHolder; +import org.apache.drill.exec.expr.fn.DrillSimpleFuncHolder; +import org.apache.drill.exec.expr.fn.FunctionConverter; +import org.apache.drill.exec.server.options.SystemOptionManager; + +import java.util.Set; + +public class InterpreterBuilder { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(InterpreterBuilder.class); + + + public static void main(String args[]) { + InterpreterBuilder builder = new InterpreterBuilder(); + if (args.length != 1) { + System.err.println("Usage: InterpreterBuilder targetSrcDir\n"); + System.exit(-1); + } + builder.build(DrillConfig.create(), args[0]); + } + + private void build(DrillConfig config, String targetSrcDir) { + FunctionConverter converter = new FunctionConverter(); + Set<Class<? extends DrillFunc>> providerClasses = PathScanner.scanForImplementations(DrillFunc.class, config.getStringList(ExecConstants.FUNCTION_PACKAGES)); + + int count = 0; + for (Class<? extends DrillFunc> clazz : providerClasses) { + try { + DrillFuncHolder holder = converter.getHolder(clazz); + + if (holder != null && holder instanceof DrillSimpleFuncHolder) { + InterpreterGenerator generator = new InterpreterGenerator((DrillSimpleFuncHolder)holder, clazz.getSimpleName() + InterpreterGenerator.INTERPRETER_CLASSNAME_POSTFIX, targetSrcDir); + generator.build(); + count ++; + } + } catch (Exception ex) { + failure("Failure while creating function interpreter.", ex, clazz); + } + } + + logger.debug("Total interpreter class generated : " + count); + } + + private void failure(String message, Throwable t, Class<?> clazz) { + logger.error("Failure loading function class [{}]. Message: {}", clazz.getName(), message, t); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9bc71fc5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterEvaluator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterEvaluator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterEvaluator.java new file mode 100644 index 0000000..0fe36cb --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterEvaluator.java @@ -0,0 +1,334 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr.fn.interpreter; + +import com.google.common.base.Preconditions; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.BooleanOperator; +import org.apache.drill.common.expression.FunctionHolderExpression; +import org.apache.drill.common.expression.IfExpression; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.ValueExpressions; +import org.apache.drill.common.expression.visitors.AbstractExprVisitor; +import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.exec.expr.DrillFuncHolderExpr; +import org.apache.drill.exec.expr.TypeHelper; +import org.apache.drill.exec.expr.ValueVectorReadExpression; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.fn.DrillSimpleFuncHolder; +import org.apache.drill.exec.expr.holders.BitHolder; +import org.apache.drill.exec.expr.holders.NullableBitHolder; +import org.apache.drill.exec.expr.holders.ValueHolder; +import org.apache.drill.exec.record.RecordBatch; +import org.apache.drill.exec.vector.ValueHolderHelper; +import org.apache.drill.exec.vector.ValueVector; + + +public class InterpreterEvaluator { + + public static void evaluate(RecordBatch incoming, ValueVector outVV, LogicalExpression expr) { + + InterpreterInitVisitor initVisitor = new InterpreterInitVisitor(); + InterEvalVisitor evalVisitor = new InterEvalVisitor(incoming); + + expr.accept(initVisitor, incoming); + + for (int i = 0; i < incoming.getRecordCount(); i++) { + ValueHolder out = expr.accept(evalVisitor, i); + TypeHelper.setValueSafe(outVV, i, out); + } + + outVV.getMutator().setValueCount(incoming.getRecordCount()); + } + + public static class InterpreterInitVisitor extends AbstractExprVisitor<LogicalExpression, RecordBatch, RuntimeException> { + @Override + public LogicalExpression visitFunctionHolderExpression(FunctionHolderExpression holderExpr, RecordBatch incoming) { + if (! (holderExpr.getHolder() instanceof DrillSimpleFuncHolder)) { + throw new UnsupportedOperationException("Only Drill simple UDF can be used in interpreter mode!"); + } + + DrillSimpleFuncHolder holder = (DrillSimpleFuncHolder) holderExpr.getHolder(); + + for (int i = 0; i < holderExpr.args.size(); i++) { + holderExpr.args.get(i).accept(this, incoming); + } + + try { + DrillSimpleFuncInterpreter interpreter = holder.createInterpreter(); + + ((DrillFuncHolderExpr) holderExpr).setInterpreter(interpreter); + + return holderExpr; + + } catch (Exception ex) { + throw new RuntimeException("Error in evaluating function of " + holderExpr.getName() + ": " + ex); + } + } + + @Override + public LogicalExpression visitUnknown(LogicalExpression e, RecordBatch incoming) throws RuntimeException { + for (LogicalExpression child : e) { + child.accept(this, incoming); + } + + return e; + } + } + + + public static class InterEvalVisitor extends AbstractExprVisitor<ValueHolder, Integer, RuntimeException> { + private RecordBatch incoming; + + protected InterEvalVisitor(RecordBatch incoming) { + super(); + this.incoming = incoming; + } + + @Override + public ValueHolder visitFunctionHolderExpression(FunctionHolderExpression holderExpr, Integer inIndex) { + if (! (holderExpr.getHolder() instanceof DrillSimpleFuncHolder)) { + throw new UnsupportedOperationException("Only Drill simple UDF can be used in interpreter mode!"); + } + + DrillSimpleFuncHolder holder = (DrillSimpleFuncHolder) holderExpr.getHolder(); + + ValueHolder [] args = new ValueHolder [holderExpr.args.size()]; + for (int i = 0; i < holderExpr.args.size(); i++) { + args[i] = holderExpr.args.get(i).accept(this, inIndex); + // In case function use "NULL_IF_NULL" policy. + if (holder.getNullHandling() == FunctionTemplate.NullHandling.NULL_IF_NULL) { + // Case 1: parameter is non-nullable, argument is nullable. + if (holder.getParameters()[i].getType().getMode() == TypeProtos.DataMode.REQUIRED && TypeHelper.getValueHolderType(args[i]).getMode() == TypeProtos.DataMode.OPTIONAL) { + // Case 1.1 : argument is null, return null value holder directly. + if (TypeHelper.isNull(args[i])) { + return TypeHelper.createValueHolder(holderExpr.getMajorType()); + } else { + // Case 1.2: argument is nullable but not null value, deNullify it. + args[i] = TypeHelper.deNullify(args[i]); + } + } else if (holder.getParameters()[i].getType().getMode() == TypeProtos.DataMode.OPTIONAL && TypeHelper.getValueHolderType(args[i]).getMode() == TypeProtos.DataMode.REQUIRED) { + // Case 2: parameter is nullable, argument is non-nullable. Nullify it. + args[i] = TypeHelper.nullify(args[i]); + } + } + } + + try { + DrillSimpleFuncInterpreter interpreter = ((DrillFuncHolderExpr) holderExpr).getInterpreter(); + + Preconditions.checkArgument(interpreter != null, "interpreter could not be null when use interpreted model to evaluate function " + holder.getRegisteredNames()[0]); + + interpreter.doSetup(args, incoming); + ValueHolder out = interpreter.doEval(args); + + if (TypeHelper.getValueHolderType(out).getMode() == TypeProtos.DataMode.OPTIONAL && + holderExpr.getMajorType().getMode() == TypeProtos.DataMode.REQUIRED) { + return TypeHelper.deNullify(out); + } else if (TypeHelper.getValueHolderType(out).getMode() == TypeProtos.DataMode.REQUIRED && + holderExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) { + return TypeHelper.nullify(out); + } else { + return out; + } + + } catch (Exception ex) { + throw new RuntimeException("Error in evaluating function of " + holderExpr.getName() + ": " + ex); + } + + } + + @Override + public ValueHolder visitBooleanOperator(BooleanOperator op, Integer inIndex) { + // Apply short circuit evaluation to boolean operator. + if (op.getName().equals("booleanAnd")) { + return visitBooleanAnd(op, inIndex); + }else if(op.getName().equals("booleanOr")) { + return visitBooleanOr(op, inIndex); + } else { + throw new UnsupportedOperationException("BooleanOperator can only be booleanAnd, booleanOr. You are using " + op.getName()); + } + } + + @Override + public ValueHolder visitIfExpression(IfExpression ifExpr, Integer inIndex) throws RuntimeException { + ValueHolder condHolder = ifExpr.ifCondition.condition.accept(this, inIndex); + + assert (condHolder instanceof BitHolder || condHolder instanceof NullableBitHolder); + + Trivalent flag = isBitOn(condHolder); + + switch (flag) { + case TRUE: + return ifExpr.ifCondition.expression.accept(this, inIndex); + case FALSE: + case NULL: + return ifExpr.elseExpression.accept(this, inIndex); + default: + throw new UnsupportedOperationException("No other possible choice. Something is not right"); + } + } + + @Override + public ValueHolder visitIntConstant(ValueExpressions.IntExpression e, Integer inIndex) throws RuntimeException { + return ValueHolderHelper.getIntHolder(e.getInt()); + } + + @Override + public ValueHolder visitFloatConstant(ValueExpressions.FloatExpression fExpr, Integer value) throws RuntimeException { + return ValueHolderHelper.getFloat4Holder(fExpr.getFloat()); + } + + @Override + public ValueHolder visitLongConstant(ValueExpressions.LongExpression intExpr, Integer value) throws RuntimeException { + return ValueHolderHelper.getBigIntHolder(intExpr.getLong()); + } + + @Override + public ValueHolder visitDoubleConstant(ValueExpressions.DoubleExpression dExpr, Integer value) throws RuntimeException { + return ValueHolderHelper.getFloat8Holder(dExpr.getDouble()); + } + + @Override + public ValueHolder visitQuotedStringConstant(ValueExpressions.QuotedString e, Integer value) throws RuntimeException { + return ValueHolderHelper.getVarCharHolder((incoming).getContext().getManagedBuffer(), e.value); + } + + + @Override + public ValueHolder visitUnknown(LogicalExpression e, Integer inIndex) throws RuntimeException { + if (e instanceof ValueVectorReadExpression) { + return visitValueVectorReadExpression((ValueVectorReadExpression) e, inIndex); + } else { + return super.visitUnknown(e, inIndex); + } + + } + + protected ValueHolder visitValueVectorReadExpression(ValueVectorReadExpression e, Integer inIndex) + throws RuntimeException { + TypeProtos.MajorType type = e.getMajorType(); + + ValueVector vv; + ValueHolder holder; + try { + switch (type.getMode()) { + case OPTIONAL: + case REQUIRED: + vv = incoming.getValueAccessorById(TypeHelper.getValueVectorClass(type.getMinorType(),type.getMode()), e.getFieldId().getFieldIds()).getValueVector(); + holder = TypeHelper.getValue(vv, inIndex.intValue()); + return holder; + default: + throw new UnsupportedOperationException("Type of " + type + " is not supported yet!"); + } + } catch (Exception ex){ + throw new DrillRuntimeException("Error when evaluate a ValueVectorReadExpression: " + ex); + } + } + + // Use Kleene algebra for three-valued logic : + // value of boolean "and" when one side is null + // p q p and q + // true null null + // false null false + // null true null + // null false false + // null null null + // "and" : 1) if any argument is false, return false. false is earlyExitValue. + // 2) if none argument is false, but at least one is null, return null. + // 3) finally, return true (finalValue). + private ValueHolder visitBooleanAnd(BooleanOperator op, Integer inIndex) { + ValueHolder [] args = new ValueHolder [op.args.size()]; + boolean hasNull = false; + ValueHolder out = null; + for (int i = 0; i < op.args.size(); i++) { + args[i] = op.args.get(i).accept(this, inIndex); + + Trivalent flag = isBitOn(args[i]); + + switch (flag) { + case FALSE: + return op.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL? TypeHelper.nullify(ValueHolderHelper.getBitHolder(0)) : ValueHolderHelper.getBitHolder(0); + case NULL: + hasNull = true; + case TRUE: + } + } + + if (hasNull) { + return ValueHolderHelper.getNullableBitHolder(true, 0); + } else { + return op.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL? TypeHelper.nullify(ValueHolderHelper.getBitHolder(1)) : ValueHolderHelper.getBitHolder(1); + } + } + + // value of boolean "or" when one side is null + // p q p and q + // true null true + // false null null + // null true true + // null false null + // null null null + private ValueHolder visitBooleanOr(BooleanOperator op, Integer inIndex) { + ValueHolder [] args = new ValueHolder [op.args.size()]; + boolean hasNull = false; + ValueHolder out = null; + for (int i = 0; i < op.args.size(); i++) { + args[i] = op.args.get(i).accept(this, inIndex); + + Trivalent flag = isBitOn(args[i]); + + switch (flag) { + case TRUE: + return op.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL? TypeHelper.nullify(ValueHolderHelper.getBitHolder(1)) : ValueHolderHelper.getBitHolder(1); + case NULL: + hasNull = true; + case FALSE: + } + } + + if (hasNull) { + return ValueHolderHelper.getNullableBitHolder(true, 0); + } else { + return op.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL? TypeHelper.nullify(ValueHolderHelper.getBitHolder(0)) : ValueHolderHelper.getBitHolder(0); + } + } + + public enum Trivalent { + FALSE, + TRUE, + NULL + } + + private Trivalent isBitOn(ValueHolder holder) { + assert (holder instanceof BitHolder || holder instanceof NullableBitHolder); + + if ( (holder instanceof BitHolder && ((BitHolder) holder).value == 1)) { + return Trivalent.TRUE; + } else if (holder instanceof NullableBitHolder && ((NullableBitHolder) holder).isSet == 1 && ((NullableBitHolder) holder).value == 1) { + return Trivalent.TRUE; + } else if (holder instanceof NullableBitHolder && ((NullableBitHolder) holder).isSet == 0) { + return Trivalent.NULL; + } else { + return Trivalent.FALSE; + } + } + } + +} + http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9bc71fc5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterGenerator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterGenerator.java new file mode 100644 index 0000000..6cede33 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterGenerator.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr.fn.interpreter; + +import com.google.common.collect.Maps; +import com.sun.codemodel.JBlock; +import com.sun.codemodel.JClass; +import com.sun.codemodel.JCodeModel; +import com.sun.codemodel.JDefinedClass; +import com.sun.codemodel.JExpr; +import com.sun.codemodel.JFieldVar; +import com.sun.codemodel.JMethod; +import com.sun.codemodel.JMod; +import com.sun.codemodel.JType; +import com.sun.codemodel.JVar; +import org.apache.drill.exec.expr.TypeHelper; +import org.apache.drill.exec.expr.fn.DrillFuncHolder; +import org.apache.drill.exec.expr.fn.DrillSimpleFuncHolder; +import org.apache.drill.exec.expr.holders.ValueHolder; +import org.apache.drill.exec.record.RecordBatch; + +import java.io.File; +import java.io.IOException; +import java.util.Map; + +public class InterpreterGenerator { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(InterpreterGenerator.class); + + public static final String PACKAGE_NAME = "org.apache.drill.exec.fn.interpreter.generated"; + private static final String ARG_NAME = "args"; + private static final String SETUP_METHOD = "doSetup"; + private static final String EVAL_METHOD = "doEval"; + public static final String INTERPRETER_CLASSNAME_POSTFIX = "Interpreter"; + + private JCodeModel model ; + private DrillSimpleFuncHolder holder; + private JDefinedClass clazz; + private String className; + private String genSourceDir; + + public InterpreterGenerator (DrillSimpleFuncHolder holder, String className, String genSourceDir) { + this.model = new JCodeModel(); + this.holder = holder; + this.className = className; + this.genSourceDir = genSourceDir; + } + + public void build() throws Exception { + try { + this.clazz = model._class(PACKAGE_NAME + "." + className); + JClass iface = model.ref(DrillSimpleFuncInterpreter.class); + clazz._implements(iface); + + Map<DrillFuncHolder.WorkspaceReference, JFieldVar> wsFieldVars = Maps.newHashMap(); + for (DrillFuncHolder.WorkspaceReference ws : holder.getWorkspaceVars()) { + JFieldVar wsVar = clazz.field(JMod.PRIVATE, ws.getType(), ws.getName()); + wsFieldVars.put(ws, wsVar); + } + + generateSetup(wsFieldVars); + + generateEval(); + + // generate the java source code for the function. + File file = new File(genSourceDir); + if (!file.exists()) { + file.mkdir(); + } + + model.build(file); + + } catch (Exception e) { + logger.error("Error when generate code for " + className + " error " + e); + throw new IllegalStateException(e); + } + } + + private void generateSetup(Map<DrillFuncHolder.WorkspaceReference, JFieldVar> wsFieldVars) { + JClass valueholderClass = model.ref(ValueHolder.class); + + JMethod doSetupMethod = clazz.method(JMod.PUBLIC, Void.TYPE, SETUP_METHOD); + doSetupMethod.param(valueholderClass.array(), ARG_NAME); + JVar incomingJVar = doSetupMethod.param(model.ref(RecordBatch.class), "incoming"); + + if (holder.getSetupBody()!=null && ! holder.getSetupBody().trim().equals("{}")) { + declareAssignParm(model, doSetupMethod.body(), holder, ARG_NAME, true); + } + + for (DrillFuncHolder.WorkspaceReference ws : holder.getWorkspaceVars()) { + if (ws.isInject()) { + doSetupMethod.body().assign(wsFieldVars.get(ws), incomingJVar.invoke("getContext").invoke("getManagedBuffer")); + } + } + + doSetupMethod.body().directStatement(holder.getSetupBody()); + } + + private void generateEval() { + JClass valueholderClass = model.ref(ValueHolder.class); + + JMethod doEvalMethod = clazz.method(JMod.PUBLIC, ValueHolder.class, EVAL_METHOD); + doEvalMethod.param(valueholderClass.array(), ARG_NAME); + + if (holder.getEvalBody()!=null && ! holder.getEvalBody().trim().equals("")) { + declareAssignParm(model, doEvalMethod.body(), holder, ARG_NAME, false); + } + + DrillFuncHolder.ValueReference returnValue = holder.getReturnValue(); + JType outType = TypeHelper.getHolderType(model, returnValue.getType().getMinorType(), returnValue.getType().getMode()); + + JVar outVar = doEvalMethod.body().decl(outType, returnValue.getName(), JExpr._new(outType)); + + doEvalMethod.body().directStatement(holder.getEvalBody()); + doEvalMethod.body()._return(outVar); + } + + private void declareAssignParm(JCodeModel model, JBlock block, DrillFuncHolder holder, String argName, boolean constantOnly) { + + int index = 0; + DrillFuncHolder.ValueReference[] parameters = holder.getParameters(); + + for (DrillFuncHolder.ValueReference parm : parameters) { + JType type = TypeHelper.getHolderType(model, parm.getType().getMinorType(), parm.getType().getMode()); + block.decl(type, parm.getName(), JExpr.cast(type, JExpr.component(JExpr.ref(argName), JExpr.lit(index++)))); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9bc71fc5/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java index 31afb6f..2f5bf6a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java @@ -21,11 +21,17 @@ import io.netty.buffer.DrillBuf; import java.math.BigDecimal; +import org.apache.drill.exec.expr.holders.BigIntHolder; +import org.apache.drill.exec.expr.holders.BitHolder; import org.apache.drill.exec.expr.holders.Decimal18Holder; import org.apache.drill.exec.expr.holders.Decimal28SparseHolder; import org.apache.drill.exec.expr.holders.Decimal38SparseHolder; import org.apache.drill.exec.expr.holders.Decimal9Holder; +import org.apache.drill.exec.expr.holders.Float4Holder; +import org.apache.drill.exec.expr.holders.Float8Holder; +import org.apache.drill.exec.expr.holders.IntHolder; import org.apache.drill.exec.expr.holders.IntervalDayHolder; +import org.apache.drill.exec.expr.holders.NullableBitHolder; import org.apache.drill.exec.expr.holders.VarCharHolder; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.util.DecimalUtility; @@ -36,6 +42,52 @@ import com.google.common.base.Charsets; public class ValueHolderHelper { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueHolderHelper.class); + public static IntHolder getIntHolder(int value) { + IntHolder holder = new IntHolder(); + holder.value = value; + + return holder; + } + + public static BigIntHolder getBigIntHolder(long value) { + BigIntHolder holder = new BigIntHolder(); + holder.value = value; + + return holder; + } + + public static Float4Holder getFloat4Holder(float value) { + Float4Holder holder = new Float4Holder(); + holder.value = value; + + return holder; + } + + public static Float8Holder getFloat8Holder(double value) { + Float8Holder holder = new Float8Holder(); + holder.value = value; + + return holder; + } + + + public static BitHolder getBitHolder(int value) { + BitHolder holder = new BitHolder(); + holder.value = value; + + return holder; + } + + public static NullableBitHolder getNullableBitHolder(boolean isNull, int value) { + NullableBitHolder holder = new NullableBitHolder(); + holder.isSet = isNull? 0 : 1; + if (! isNull) { + holder.value = value; + } + + return holder; + } + public static VarCharHolder getVarCharHolder(DrillBuf buf, String s){ VarCharHolder vch = new VarCharHolder(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9bc71fc5/exec/pom.xml ---------------------------------------------------------------------- diff --git a/exec/pom.xml b/exec/pom.xml index fe2a75e..79ed482 100644 --- a/exec/pom.xml +++ b/exec/pom.xml @@ -34,5 +34,6 @@ <module>java-exec</module> <module>jdbc</module> <module>jdbc-all</module> + <module>interpreter</module> </modules> </project>
