http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java index e91b904..0c9b979 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java @@ -1,12 +1,10 @@ package org.apache.drill.exec.expr.fn; -import java.util.Arrays; import java.util.List; import java.util.Map; import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.Types; import org.apache.drill.exec.expr.CodeGenerator; @@ -16,29 +14,25 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.sun.codemodel.JBlock; -import com.sun.codemodel.JConditional; -import com.sun.codemodel.JExpr; -import com.sun.codemodel.JExpression; import com.sun.codemodel.JMod; import com.sun.codemodel.JVar; -public class FunctionHolder { +public abstract class FunctionHolder { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionImplementationRegistry.class); - private FunctionTemplate.FunctionScope scope; - private FunctionTemplate.NullHandling nullHandling; - private boolean isBinaryCommutative; - private String functionName; - private String evalBody; - private String addBody; - private String setupBody; - private List<String> imports; - private WorkspaceReference[] workspaceVars; - private ValueReference[] parameters; - private ValueReference returnValue; + protected final FunctionTemplate.FunctionScope scope; + protected final FunctionTemplate.NullHandling nullHandling; + protected final boolean isBinaryCommutative; + protected final String functionName; + protected final ImmutableList<String> imports; + protected final WorkspaceReference[] workspaceVars; + protected final ValueReference[] parameters; + protected final ValueReference returnValue; + protected final ImmutableMap<String, String> methodMap; public FunctionHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, String functionName, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, Map<String, String> methods, List<String> imports) { super(); @@ -47,14 +41,10 @@ public class FunctionHolder { this.workspaceVars = workspaceVars; this.isBinaryCommutative = isBinaryCommutative; this.functionName = functionName; - this.setupBody = methods.get("setup"); - this.addBody = methods.get("add"); - this.evalBody = methods.get("eval"); - Preconditions.checkNotNull(evalBody); - Preconditions.checkArgument(!evalBody.isEmpty()); + this.methodMap = ImmutableMap.copyOf(methods); this.parameters = parameters; this.returnValue = returnValue; - this.imports = imports; + this.imports = ImmutableList.copyOf(imports); } @@ -62,15 +52,7 @@ public class FunctionHolder { return imports; } - private void generateSetupBody(CodeGenerator<?> g){ - if(!Strings.isNullOrEmpty(setupBody)){ - JBlock sub = new JBlock(true, true); - addProtectedBlock(g, sub, setupBody, null); - g.getSetupBlock().directStatement(String.format("/** start setup for function %s **/", functionName)); - g.getSetupBlock().add(sub); - g.getSetupBlock().directStatement(String.format("/** end setup for function %s **/", functionName)); - } - } + public abstract HoldingContainer renderFunction(CodeGenerator<?> g, HoldingContainer[] inputVariables); public void addProtectedBlock(CodeGenerator<?> g, JBlock sub, String body, HoldingContainer[] inputVariables){ @@ -102,57 +84,7 @@ public class FunctionHolder { } } - public HoldingContainer renderFunction(CodeGenerator<?> g, HoldingContainer[] inputVariables){ - generateSetupBody(g); - return generateEvalBody(g, inputVariables); - } - - private HoldingContainer generateEvalBody(CodeGenerator<?> g, HoldingContainer[] inputVariables){ - - //g.getBlock().directStatement(String.format("//---- start of eval portion of %s function. ----//", functionName)); - - JBlock sub = new JBlock(true, true); - JBlock topSub = sub; - HoldingContainer out = null; - - // add outside null handling if it is defined. - if(nullHandling == NullHandling.NULL_IF_NULL){ - JExpression e = null; - for(HoldingContainer v : inputVariables){ - if(v.isOptional()){ - if(e == null){ - e = v.getIsSet(); - }else{ - e = e.mul(v.getIsSet()); - } - } - } - - if(e != null){ - // if at least one expression must be checked, set up the conditional. - returnValue.type = returnValue.type.toBuilder().setMode(DataMode.OPTIONAL).build(); - out = g.declare(returnValue.type); - e = e.eq(JExpr.lit(0)); - JConditional jc = sub._if(e); - jc._then().assign(out.getIsSet(), JExpr.lit(0)); - sub = jc._else(); - } - } - - if(out == null) out = g.declare(returnValue.type); - - // add the subblock after the out declaration. - g.getBlock().add(topSub); - - - JVar internalOutput = sub.decl(JMod.FINAL, g.getHolderType(returnValue.type), returnValue.name, JExpr._new(g.getHolderType(returnValue.type))); - addProtectedBlock(g, sub, evalBody, inputVariables); - if (sub != topSub) sub.assign(internalOutput.ref("isSet"),JExpr.lit(1));// Assign null if NULL_IF_NULL mode - sub.assign(out.getHolder(), internalOutput); - - return out; - } - + public boolean matches(FunctionCall call){ @@ -215,15 +147,7 @@ public class FunctionHolder { } } - @Override - public String toString() { - final int maxLen = 10; - return "FunctionHolder [scope=" + scope + ", isBinaryCommutative=" + isBinaryCommutative + ", functionName=" - + functionName + ", evalBody=" + evalBody + ", addBody=" + addBody + ", setupBody=" + setupBody - + ", parameters=" - + (parameters != null ? Arrays.asList(parameters).subList(0, Math.min(parameters.length, maxLen)) : null) - + ", returnValue=" + returnValue + "]"; - } + }
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java index c2dfca4..bbe4cfb 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java @@ -107,14 +107,6 @@ public class FragmentContext { return t; } - - public <T> T getImplementationClassMultipleOutput(CodeGenerator<T> cg) throws ClassTransformationException, IOException{ - long t1 = System.nanoTime(); - T t = transformer.getImplementationClass(this.loader, cg.getDefinition(), cg.generateMultipleOutputs(), cg.getMaterializedClassName()); - logger.debug("Compile time: {} millis.", (System.nanoTime() - t1)/1000/1000 ); - return t; - - } public void addMetricsToStatus(FragmentStatus.Builder stats){ stats.setBatchesCompleted(batchesCompleted.get()); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterSignature.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterSignature.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterSignature.java new file mode 100644 index 0000000..f085cd3 --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterSignature.java @@ -0,0 +1,14 @@ +package org.apache.drill.exec.physical.impl.filter; + +import javax.inject.Named; + +import org.apache.drill.exec.compile.sig.CodeGeneratorSignature; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.record.RecordBatch; + +public interface FilterSignature extends CodeGeneratorSignature{ + + public void doSetup(@Named("context") FragmentContext context, @Named("incoming") RecordBatch incoming, @Named("outgoing") RecordBatch outgoing); + public boolean doEval(@Named("inIndex") int inIndex, @Named("outIndex") int outIndex); + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/Filterer.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/Filterer.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/Filterer.java index b270869..fb08ef3 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/Filterer.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/Filterer.java @@ -3,7 +3,6 @@ package org.apache.drill.exec.physical.impl.filter; import org.apache.drill.exec.compile.TemplateClassDefinition; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.ops.FragmentContext; -import org.apache.drill.exec.physical.impl.project.Projector; import org.apache.drill.exec.record.RecordBatch; import org.apache.drill.exec.record.TransferPair; @@ -14,6 +13,6 @@ public interface Filterer { public void filterBatch(int recordCount); public static TemplateClassDefinition<Filterer> TEMPLATE_DEFINITION = new TemplateClassDefinition<Filterer>( // - Filterer.class, "org.apache.drill.exec.physical.impl.filter.FilterTemplate", FilterEvaluator.class, boolean.class); + Filterer.class, "org.apache.drill.exec.physical.impl.filter.FilterTemplate", FilterEvaluator.class, FilterSignature.class); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ReturnValueExpression.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ReturnValueExpression.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ReturnValueExpression.java index a794d63..cfe520d 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ReturnValueExpression.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ReturnValueExpression.java @@ -1,11 +1,15 @@ package org.apache.drill.exec.physical.impl.filter; +import java.util.Iterator; + import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.visitors.ExprVisitor; import org.apache.drill.common.types.Types; import org.apache.drill.common.types.TypeProtos.MajorType; +import com.google.common.collect.Iterators; + public class ReturnValueExpression implements LogicalExpression{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ReturnValueExpression.class); @@ -34,6 +38,10 @@ public class ReturnValueExpression implements LogicalExpression{ return ExpressionPosition.UNKNOWN; } - + @Override + public Iterator<LogicalExpression> iterator() { + return Iterators.singletonIterator(child); + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java index 6d24e0b..f96e3cf 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java @@ -171,7 +171,7 @@ class PartitionSenderRootExec implements RootExec { // generate evaluate expression to determine the hash CodeGenerator.HoldingContainer exprHolder = cg.addExpr(materializedExpr); - cg.getBlock().decl(JType.parse(cg.getModel(), "int"), "bucket", exprHolder.getValue().mod(JExpr.lit(outgoing.length))); + cg.getEvalBlock().decl(JType.parse(cg.getModel(), "int"), "bucket", exprHolder.getValue().mod(JExpr.lit(outgoing.length))); // declare and assign the array of outgoing record batches JVar outgoingBatches = cg.clazz.field(JMod.NONE, @@ -230,7 +230,7 @@ class PartitionSenderRootExec implements RootExec { // ((IntVector) outgoingVectors[bucket][0]).copyFrom(inIndex, // outgoingBatches[bucket].getRecordCount(), // vv1); - cg.getBlock().add( + cg.getEvalBlock().add( ((JExpression) JExpr.cast(vvClass, ((JExpression) outgoingVectors @@ -244,11 +244,12 @@ class PartitionSenderRootExec implements RootExec { ++fieldId; } // generate the OutgoingRecordBatch helper invocations - cg.getBlock().add(((JExpression) outgoingBatches.component(bucket)).invoke("incRecordCount")); - cg.getBlock().add(((JExpression) outgoingBatches.component(bucket)).invoke("flushIfNecessary")); + cg.getEvalBlock().add(((JExpression) outgoingBatches.component(bucket)).invoke("incRecordCount")); + cg.getEvalBlock().add(((JExpression) outgoingBatches.component(bucket)).invoke("flushIfNecessary")); try { // compile and setup generated code - partitioner = context.getImplementationClassMultipleOutput(cg); +// partitioner = context.getImplementationClassMultipleOutput(cg); + partitioner = context.getImplementationClass(cg); partitioner.setup(context, incoming, outgoing); } catch (ClassTransformationException | IOException e) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/Partitioner.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/Partitioner.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/Partitioner.java index e8f2ca7..992ffdf 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/Partitioner.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/Partitioner.java @@ -35,5 +35,5 @@ public interface Partitioner { new TemplateClassDefinition<>(Partitioner.class, "org.apache.drill.exec.physical.impl.partitionsender.PartitionerTemplate", PartitionerEvaluator.class, - null); + PartitionerInnerSignature.class); } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerInnerSignature.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerInnerSignature.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerInnerSignature.java new file mode 100644 index 0000000..be209a9 --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerInnerSignature.java @@ -0,0 +1,17 @@ +package org.apache.drill.exec.physical.impl.partitionsender; + +import javax.inject.Named; + +import org.apache.drill.exec.compile.sig.CodeGeneratorSignature; +import org.apache.drill.exec.exception.SchemaChangeException; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.record.RecordBatch; + +public interface PartitionerInnerSignature extends CodeGeneratorSignature{ + + public void doSetup(@Named("context") FragmentContext context, @Named("incoming") RecordBatch incoming, @Named("outgoing") OutgoingRecordBatch[] outgoing) throws SchemaChangeException; + public void doEval(@Named("inIndex") int inIndex, @Named("outIndex") int outIndex); + + + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java index 5fd1fb4..75632e7 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java @@ -8,5 +8,5 @@ public interface ProjectEvaluator { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProjectEvaluator.class); public abstract void doSetup(FragmentContext context, RecordBatch incoming, RecordBatch outgoing) throws SchemaChangeException; - public abstract void doEval(int inIndex, int outIndex); + public abstract void doEval(int inIndex, int outIndex) throws SchemaChangeException; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java index 0d1e201..ba83e61 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java @@ -3,6 +3,7 @@ package org.apache.drill.exec.physical.impl.project; import java.util.List; import org.apache.drill.exec.compile.TemplateClassDefinition; +import org.apache.drill.exec.compile.sig.DefaultGeneratorSignature; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.record.RecordBatch; @@ -16,6 +17,6 @@ public interface Projector { public abstract int projectRecords(int recordCount, int firstOutputIndex); public static TemplateClassDefinition<Projector> TEMPLATE_DEFINITION = new TemplateClassDefinition<Projector>( // - Projector.class, "org.apache.drill.exec.physical.impl.project.ProjectorTemplate", ProjectEvaluator.class, null); + Projector.class, "org.apache.drill.exec.physical.impl.project.ProjectorTemplate", ProjectEvaluator.class); } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/ReadIndexRewriter.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/ReadIndexRewriter.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/ReadIndexRewriter.java index 83d43b2..02fffa5 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/ReadIndexRewriter.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/ReadIndexRewriter.java @@ -45,7 +45,7 @@ public class ReadIndexRewriter implements ExprVisitor<LogicalExpression, String, @Override public LogicalExpression visitIfExpression(IfExpression ifExpr, String newIndexName) { - List<IfExpression.IfCondition> conditions = Lists.newArrayList(ifExpr.iterator()); + List<IfExpression.IfCondition> conditions = Lists.newArrayList(ifExpr.conditions); LogicalExpression newElseExpr = ifExpr.elseExpression.accept(this, null); for (int i = 0; i < conditions.size(); ++i) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java index e361e38..c9bd55d 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java @@ -14,6 +14,7 @@ import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.expr.CodeGenerator; import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer; import org.apache.drill.exec.expr.ExpressionTreeMaterializer; +import org.apache.drill.exec.expr.HoldingContainerExpression; import org.apache.drill.exec.expr.fn.impl.ComparatorFunctions; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.config.Sort; @@ -123,22 +124,26 @@ public class SortBatch extends AbstractRecordBatch<Sort> { } + private Sorter createNewSorter() throws ClassTransformationException, IOException, SchemaChangeException{ CodeGenerator<Sorter> g = new CodeGenerator<Sorter>(Sorter.TEMPLATE_DEFINITION, context.getFunctionRegistry()); + g.setMappingSet(SortSignature.MAIN_MAPPING); for(OrderDef od : popConfig.getOrderings()){ // first, we rewrite the evaluation stack for each side of the comparison. ErrorCollector collector = new ErrorCollectorImpl(); final LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), this, collector); if(collector.hasErrors()) throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString()); - ReadIndexRewriter rewriter = new ReadIndexRewriter(); - LogicalExpression left = expr.accept(rewriter, "inIndex"); - LogicalExpression right = expr.accept(rewriter, "outIndex"); + g.setMappingSet(SortSignature.LEFT_MAPPING); + HoldingContainer left = g.addExpr(expr, false); + g.setMappingSet(SortSignature.RIGHT_MAPPING); + HoldingContainer right = g.addExpr(expr, false); + g.setMappingSet(SortSignature.MAIN_MAPPING); // next we wrap the two comparison sides and add the expression block for the comparison. - FunctionCall f = new FunctionCall(ComparatorFunctions.COMPARE_TO, ImmutableList.of(left, right), ExpressionPosition.UNKNOWN); - HoldingContainer out = g.addExpr(f); - JConditional jc = g.getBlock()._if(out.getValue().ne(JExpr.lit(0))); + FunctionCall f = new FunctionCall(ComparatorFunctions.COMPARE_TO, ImmutableList.of((LogicalExpression) new HoldingContainerExpression(left), new HoldingContainerExpression(right)), ExpressionPosition.UNKNOWN); + HoldingContainer out = g.addExpr(f, false); + JConditional jc = g.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0))); //TODO: is this the right order... if(od.getDirection() == Direction.ASC){ @@ -148,7 +153,7 @@ public class SortBatch extends AbstractRecordBatch<Sort> { } } - g.getBlock()._return(JExpr.lit(0)); + g.getEvalBlock()._return(JExpr.lit(0)); return context.getImplementationClass(g); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortSignature.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortSignature.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortSignature.java new file mode 100644 index 0000000..7614f3e --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortSignature.java @@ -0,0 +1,20 @@ +package org.apache.drill.exec.physical.impl.sort; + +import javax.inject.Named; + +import org.apache.drill.exec.compile.sig.CodeGeneratorSignature; +import org.apache.drill.exec.compile.sig.DefaultGeneratorSignature; +import org.apache.drill.exec.compile.sig.MappingSet; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.record.RecordBatch; + +public interface SortSignature extends CodeGeneratorSignature{ + + public static final MappingSet MAIN_MAPPING = new MappingSet("null", "null", DefaultGeneratorSignature.DEFAULT_SCALAR_MAP, DefaultGeneratorSignature.DEFAULT_SCALAR_MAP); + public static final MappingSet LEFT_MAPPING = new MappingSet("leftIndex", "null", DefaultGeneratorSignature.DEFAULT_SCALAR_MAP, DefaultGeneratorSignature.DEFAULT_SCALAR_MAP); + public static final MappingSet RIGHT_MAPPING = new MappingSet("rightIndex", "null", DefaultGeneratorSignature.DEFAULT_SCALAR_MAP, DefaultGeneratorSignature.DEFAULT_SCALAR_MAP); + + public void doSetup(@Named("context") FragmentContext context, @Named("incoming") RecordBatch incoming, @Named("outgoing") RecordBatch outgoing); + public int doEval(@Named("leftIndex") int leftIndex, @Named("rightIndex") int rightIndex); + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortTemplate.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortTemplate.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortTemplate.java index c45f500..d312fb4 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortTemplate.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortTemplate.java @@ -34,12 +34,12 @@ public abstract class SortTemplate implements Sorter, IndexedSortable{ } @Override - public int compare(int inIndex, int outIndex) { - int sv1 = vector4.get(inIndex); - int sv2 = vector4.get(outIndex); + public int compare(int leftIndex, int rightIndex) { + int sv1 = vector4.get(leftIndex); + int sv2 = vector4.get(rightIndex); return doEval(sv1, sv2); } public abstract void doSetup(FragmentContext context, RecordBatch incoming, RecordBatch outgoing) throws SchemaChangeException; - public abstract int doEval(int inIndex, int outIndex); + public abstract int doEval(int leftIndex, int rightIndex); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/Sorter.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/Sorter.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/Sorter.java index bc4fae5..1a76423 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/Sorter.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/Sorter.java @@ -12,6 +12,6 @@ public interface Sorter { public void sort(SelectionVector4 vector4, VectorContainer container); public static TemplateClassDefinition<Sorter> TEMPLATE_DEFINITION = new TemplateClassDefinition<Sorter>( // - Sorter.class, "org.apache.drill.exec.physical.impl.sort.SortTemplate", Comparator.class, int.class); + Sorter.class, "org.apache.drill.exec.physical.impl.sort.SortTemplate", Comparator.class, SortSignature.class); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/Copier.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/Copier.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/Copier.java index ce17a2b..363bbee 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/Copier.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/Copier.java @@ -1,18 +1,18 @@ package org.apache.drill.exec.physical.impl.svremover; import org.apache.drill.exec.compile.TemplateClassDefinition; +import org.apache.drill.exec.compile.sig.DefaultGeneratorSignature; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.impl.svremover.RemovingRecordBatch.VectorAllocator; import org.apache.drill.exec.record.RecordBatch; -import org.apache.drill.exec.record.selection.SelectionVector2; public interface Copier { public static TemplateClassDefinition<Copier> TEMPLATE_DEFINITION2 = new TemplateClassDefinition<Copier>( // - Copier.class, "org.apache.drill.exec.physical.impl.svremover.CopierTemplate2", CopyEvaluator.class, null); + Copier.class, "org.apache.drill.exec.physical.impl.svremover.CopierTemplate2", CopyEvaluator.class); public static TemplateClassDefinition<Copier> TEMPLATE_DEFINITION4 = new TemplateClassDefinition<Copier>( // - Copier.class, "org.apache.drill.exec.physical.impl.svremover.CopierTemplate4", CopyEvaluator.class, null); + Copier.class, "org.apache.drill.exec.physical.impl.svremover.CopierTemplate4", CopyEvaluator.class); public void setupRemover(FragmentContext context, RecordBatch incoming, RecordBatch outgoing, VectorAllocator[] allocators) throws SchemaChangeException; public abstract void copyRecords(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java index 64e89ee..e4fd9a0 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java @@ -167,7 +167,7 @@ public class RemovingRecordBatch extends AbstractSingleRecordBatch<SelectionVect if(hyper){ - g.getBlock().add( + g.getEvalBlock().add( outVV .invoke("copyFrom") .arg( @@ -178,7 +178,7 @@ public class RemovingRecordBatch extends AbstractSingleRecordBatch<SelectionVect ) ); }else{ - g.getBlock().add(outVV.invoke("copyFrom").arg(inIndex).arg(outIndex).arg(inVV)); + g.getEvalBlock().add(outVV.invoke("copyFrom").arg(inIndex).arg(outIndex).arg(inVV)); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/NullExpression.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/NullExpression.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/NullExpression.java index c2f2a69..821a4b4 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/NullExpression.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/NullExpression.java @@ -1,12 +1,15 @@ package org.apache.drill.exec.record; +import java.util.Iterator; + import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.visitors.ExprVisitor; -import org.apache.drill.common.types.Types; -import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.common.types.Types; + +import com.google.common.collect.Iterators; public class NullExpression implements LogicalExpression{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NullExpression.class); @@ -29,5 +32,11 @@ public class NullExpression implements LogicalExpression{ public ExpressionPosition getPosition() { return ExpressionPosition.UNKNOWN; } + + @Override + public Iterator<LogicalExpression> iterator() { + return Iterators.emptyIterator(); + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java index d2889ed..16c993d 100644 --- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java +++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java @@ -40,7 +40,7 @@ public class TestClassTransformation { TemplateClassDefinition<ExampleExternalInterface> def = new TemplateClassDefinition<ExampleExternalInterface>( ExampleExternalInterface.class, "org.apache.drill.exec.compile.ExampleTemplate", - ExampleInternalInterface.class, null); + ExampleInternalInterface.class); ClassTransformer ct = new ClassTransformer(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java index 4129079..a25e234 100644 --- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java +++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java @@ -18,8 +18,10 @@ package org.apache.drill.exec.physical.impl; -import com.google.common.base.Charsets; -import com.google.common.io.Files; +import static org.junit.Assert.assertEquals; + +import java.util.List; + import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.pop.PopUnitTestBase; @@ -27,11 +29,12 @@ import org.apache.drill.exec.proto.UserProtos; import org.apache.drill.exec.rpc.user.QueryResultBatch; import org.apache.drill.exec.server.Drillbit; import org.apache.drill.exec.server.RemoteServiceSet; +import org.junit.Ignore; import org.junit.Test; -import java.nio.charset.Charset; -import java.util.List; -import static org.junit.Assert.assertEquals; +import com.google.common.base.Charsets; +import com.google.common.io.Files; + public class TestHashToRandomExchange extends PopUnitTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestHashToRandomExchange.class); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ac8590d2/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java index 8a1736c..f0d9901 100644 --- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java +++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java @@ -30,7 +30,7 @@ import org.apache.drill.exec.proto.SchemaDefProtos.FieldDef; import org.apache.drill.exec.proto.SchemaDefProtos.NamePart; import org.junit.Test; -import com.google.common.collect.Lists; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Range; public class ExpressionTreeMaterializerTest { @@ -42,16 +42,16 @@ public class ExpressionTreeMaterializerTest { final MajorType intType = MajorType.newBuilder().setMode(DataMode.REQUIRED).setMinorType(MinorType.INT).build(); private MaterializedField getField(int fieldId, String name, MajorType type) { - return new MaterializedField(FieldDef.newBuilder().setMajorType(type) - .addName(NamePart.newBuilder().setName(name)).build()); + return new MaterializedField(FieldDef.newBuilder().setMajorType(type).addName(NamePart.newBuilder().setName(name)) + .build()); } - @Test public void testMaterializingConstantTree(@Injectable RecordBatch batch) throws SchemaChangeException { - + ErrorCollector ec = new ErrorCollectorImpl(); - LogicalExpression expr = ExpressionTreeMaterializer.materialize(new ValueExpressions.LongExpression(1L, ExpressionPosition.UNKNOWN), batch, ec); + LogicalExpression expr = ExpressionTreeMaterializer.materialize(new ValueExpressions.LongExpression(1L, + ExpressionPosition.UNKNOWN), batch, ec); assertTrue(expr instanceof ValueExpressions.LongExpression); assertEquals(1L, ValueExpressions.LongExpression.class.cast(expr).getLong()); assertFalse(ec.hasErrors()); @@ -62,13 +62,14 @@ public class ExpressionTreeMaterializerTest { final SchemaBuilder builder = BatchSchema.newBuilder(); builder.addField(getField(2, "test", bigIntType)); final BatchSchema schema = builder.build(); - + new NonStrictExpectations() { { - batch.getValueVectorId(new FieldReference("test", ExpressionPosition.UNKNOWN)); result = new TypedFieldId(Types.required(MinorType.BIGINT), -5); + batch.getValueVectorId(new FieldReference("test", ExpressionPosition.UNKNOWN)); + result = new TypedFieldId(Types.required(MinorType.BIGINT), -5); } }; - + ErrorCollector ec = new ErrorCollectorImpl(); LogicalExpression expr = ExpressionTreeMaterializer.materialize(new FieldReference("test", ExpressionPosition.UNKNOWN), batch, ec); @@ -80,23 +81,26 @@ public class ExpressionTreeMaterializerTest { public void testMaterializingLateboundTree(final @Injectable RecordBatch batch) throws SchemaChangeException { new NonStrictExpectations() { { - batch.getValueVectorId(new FieldReference("test", ExpressionPosition.UNKNOWN)); result = new TypedFieldId(Types.required(MinorType.BIT), -4); - batch.getValueVectorId(new FieldReference("test1", ExpressionPosition.UNKNOWN)); result = new TypedFieldId(Types.required(MinorType.BIGINT), -5); + batch.getValueVectorId(new FieldReference("test", ExpressionPosition.UNKNOWN)); + result = new TypedFieldId(Types.required(MinorType.BIT), -4); + batch.getValueVectorId(new FieldReference("test1", ExpressionPosition.UNKNOWN)); + result = new TypedFieldId(Types.required(MinorType.BIGINT), -5); } }; - + ErrorCollector ec = new ErrorCollectorImpl(); - - LogicalExpression expr = new IfExpression.Builder() + LogicalExpression expr = new IfExpression.Builder() .addCondition( new IfExpression.IfCondition( // new FieldReference("test", ExpressionPosition.UNKNOWN), // - new IfExpression.Builder() // + new IfExpression.Builder() + // .addCondition( // - new IfExpression.IfCondition( // - new ValueExpressions.BooleanExpression("true", ExpressionPosition.UNKNOWN), new FieldReference( - "test1", ExpressionPosition.UNKNOWN))) + new IfExpression.IfCondition( + // + new ValueExpressions.BooleanExpression("true", ExpressionPosition.UNKNOWN), + new FieldReference("test1", ExpressionPosition.UNKNOWN))) .setElse(new ValueExpressions.LongExpression(1L, ExpressionPosition.UNKNOWN)).build()) // ) // .setElse(new ValueExpressions.LongExpression(1L, ExpressionPosition.UNKNOWN)).build(); @@ -111,7 +115,8 @@ public class ExpressionTreeMaterializerTest { ifCondition = newIfExpr.conditions.get(0); assertEquals(bigIntType, ifCondition.expression.getMajorType()); assertEquals(true, ((ValueExpressions.BooleanExpression) ifCondition.condition).value); - if (ec.hasErrors()) System.out.println(ec.toErrorString()); + if (ec.hasErrors()) + System.out.println(ec.toErrorString()); assertFalse(ec.hasErrors()); } @@ -126,8 +131,8 @@ public class ExpressionTreeMaterializerTest { } @Override - public void addUnexpectedArgumentType(ExpressionPosition expr, String name, MajorType actual, MajorType[] expected, - int argumentIndex) { + public void addUnexpectedArgumentType(ExpressionPosition expr, String name, MajorType actual, + MajorType[] expected, int argumentIndex) { errorCount++; } @@ -174,14 +179,17 @@ public class ExpressionTreeMaterializerTest { new NonStrictExpectations() { { - batch.getValueVectorId(new FieldReference("test", ExpressionPosition.UNKNOWN)); result = new TypedFieldId(Types.required(MinorType.BIGINT), -5); + batch.getValueVectorId(new FieldReference("test", ExpressionPosition.UNKNOWN)); + result = new TypedFieldId(Types.required(MinorType.BIGINT), -5); } }; + LogicalExpression functionCallExpr = new FunctionCall(FunctionDefinition.simple("testFunc", new ArgumentValidator() { @Override - public void validateArguments(ExpressionPosition expr, List<LogicalExpression> expressions, ErrorCollector errors) { + public void validateArguments(ExpressionPosition expr, List<LogicalExpression> expressions, + ErrorCollector errors) { errors.addGeneralError(expr, "Error!"); } @@ -189,8 +197,8 @@ public class ExpressionTreeMaterializerTest { public String[] getArgumentNamesByPosition() { return new String[0]; } - }, OutputTypeDeterminer.FIXED_BIT), Lists.newArrayList((LogicalExpression) new FieldReference("test", - ExpressionPosition.UNKNOWN)), ExpressionPosition.UNKNOWN); + }, OutputTypeDeterminer.FIXED_BIT), ImmutableList.of((LogicalExpression) // + new FieldReference("test", ExpressionPosition.UNKNOWN) ), ExpressionPosition.UNKNOWN); LogicalExpression newExpr = ExpressionTreeMaterializer.materialize(functionCallExpr, batch, ec); assertTrue(newExpr instanceof FunctionCall); FunctionCall funcExpr = (FunctionCall) newExpr;
