Updated Branches: refs/heads/master db3afaa85 -> bd41633f1
Add Repeated MockRecordReader generators Refactor usesHolderForGet to Types class (updating evaluation visitor) Rename FunctionScopes for Aggregates to more descriptive names. Fix RepeatedValueVector off by 1 bug Add extra debugging in function holder matching Fix method grabbing visitor and modified unparse visitor for outer block drill func bug. (Ensure writer flush and close, add extra printing surrounding output of unparse visitor.) Update allocation helper to support repeated vectors Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ecb80fc2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ecb80fc2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ecb80fc2 Branch: refs/heads/master Commit: ecb80fc29b2df4a7382e49f1ac519fc1963767a2 Parents: db3afaa Author: Jacques Nadeau <[email protected]> Authored: Thu Aug 8 12:08:04 2013 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Thu Aug 8 12:08:04 2013 -0700 ---------------------------------------------------------------------- .../org/apache/drill/common/expression/Arg.java | 4 +-- .../org/apache/drill/common/types/Types.java | 35 ++++++++++++++------ .../templates/RepeatedValueVectors.java | 21 ++++++++++-- .../drill/exec/expr/EvaluationVisitor.java | 7 ++-- .../exec/expr/annotations/FunctionTemplate.java | 2 +- .../drill/exec/expr/fn/FunctionHolder.java | 17 ++++++++-- .../exec/expr/fn/MethodGrabbingVisitor.java | 6 ++++ .../exec/expr/fn/ModifiedUnparseVisitor.java | 26 +++++++++------ .../exec/physical/config/MockRecordReader.java | 7 ++-- .../drill/exec/vector/AllocationHelper.java | 8 +++++ 10 files changed, 94 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecb80fc2/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/Arg.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/Arg.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/Arg.java index c8d847a..217045c 100644 --- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/Arg.java +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/Arg.java @@ -67,11 +67,11 @@ public class Arg { if(ConstantChecker.onlyIncludesConstants(e)) errors.addExpectedConstantValue(expr, argIndex, name); } MajorType dt = e.getMajorType(); - if(dt.getMinorType() == MinorType.LATE){ + if(dt.getMinorType() != MinorType.LATE){ // change among allowed types. for(MajorType a : allowedTypes){ - if(dt == a) return; + if(dt.equals(a)) return; } // didn't find an allowed type. http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecb80fc2/sandbox/prototype/common/src/main/java/org/apache/drill/common/types/Types.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/types/Types.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/types/Types.java index 0490d8a..e81bc89 100644 --- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/types/Types.java +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/types/Types.java @@ -39,6 +39,30 @@ public class Types { } } + public static boolean usesHolderForGet(MajorType type){ + if(type.getMode() == DataMode.REPEATED) return true; + switch(type.getMinorType()){ + case BIGINT: + case DECIMAL4: + case DECIMAL8: + case FLOAT4: + case FLOAT8: + case INT: + case MONEY: + case SMALLINT: + case TINYINT: + case UINT1: + case UINT2: + case UINT4: + case UINT8: + return false; + + default: + return true; + } + + } + public static boolean isFixedWidthType(MajorType type){ switch(type.getMinorType()){ case VARBINARY: @@ -105,19 +129,10 @@ public class Types { case OPTIONAL: case REQUIRED: return true; - default: - return false; } - default: - return false; - } - }else{ - if(a.getMode() != b.getMode()){ - return false; - }else{ - return true; } } + return a.getMode() == b.getMode(); } public static boolean isLateBind(MajorType type){ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecb80fc2/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java index 79f05b6..1afe84b 100644 --- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java +++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java @@ -225,6 +225,12 @@ import com.google.common.collect.Lists; </#if> get(int index, int positionIndex) { return values.getAccessor().get(offsets.getAccessor().get(index) + positionIndex); } + + public void get(int index, Repeated${minor.class}Holder holder){ + holder.start = offsets.getAccessor().get(index); + holder.end = offsets.getAccessor().get(index+1); + holder.vector = values; + } public MaterializedField getField() { return field; @@ -288,13 +294,22 @@ import com.google.common.collect.Lists; */ public void setValueCount(int groupCount) { parentValueCount = groupCount; - childValueCount = offsets.getAccessor().get(groupCount+1); - offsets.getMutator().setValueCount(groupCount); + childValueCount = offsets.getAccessor().get(groupCount); + offsets.getMutator().setValueCount(groupCount+1); values.getMutator().setValueCount(childValueCount); } public void generateTestData(){ - throw new UnsupportedOperationException(); + setValueCount(offsets.getAccessor().getValueCount() - 1); + int valCount = offsets.getValueCapacity(); + int[] sizes = {1,2,0,6}; + int size = 0; + int runningOffset = 0; + for(int i =1; i < valCount; i++, size++){ + runningOffset += sizes[size % sizes.length]; + offsets.getMutator().set(i, runningOffset); + } + values.getMutator().generateTestData(); } public void reset(){ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecb80fc2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java index 93d5b5b..21106c1 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java @@ -11,6 +11,7 @@ import org.apache.drill.common.expression.ValueExpressions.LongExpression; import org.apache.drill.common.expression.ValueExpressions.QuotedString; import org.apache.drill.common.expression.visitors.AbstractExprVisitor; import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.common.types.Types; import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer; import org.apache.drill.exec.expr.fn.FunctionHolder; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; @@ -175,8 +176,7 @@ public class EvaluationVisitor extends AbstractExprVisitor<HoldingContainer, Cod JBlock blk = generator.getBlock(); blk.assign(out.getIsSet(), vv1.invoke("getAccessor").invoke("isSet").arg(indexVariable)); JConditional jc = blk._if(out.getIsSet().eq(JExpr.lit(1))); - if (e.getMajorType().getMinorType() == TypeProtos.MinorType.VARCHAR || - e.getMajorType().getMinorType() == TypeProtos.MinorType.VARBINARY) { + if (Types.usesHolderForGet(e.getMajorType())) { jc._then() .add(getValueAccessor.arg(JExpr.direct("inIndex")).arg(out.getHolder())); } else { @@ -184,8 +184,7 @@ public class EvaluationVisitor extends AbstractExprVisitor<HoldingContainer, Cod .assign(out.getValue(), getValueAccessor.arg(indexVariable)); } }else{ - if (e.getMajorType().getMinorType() == TypeProtos.MinorType.VARCHAR || - e.getMajorType().getMinorType() == TypeProtos.MinorType.VARBINARY) { + if (Types.usesHolderForGet(e.getMajorType())) { generator.getBlock().add(getValueAccessor.arg(indexVariable).arg(out.getHolder())); } else { generator.getBlock().assign(out.getValue(), getValueAccessor.arg(indexVariable)); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecb80fc2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java index 9e32750..2e7a458 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java @@ -20,6 +20,6 @@ public @interface FunctionTemplate { } public static enum FunctionScope{ - SIMPLE, AGGREGATE, RUNNING; + SIMPLE, POINT_AGGREGATE, HOLISTIC_AGGREGATE, RANGE_AGGREGATE; } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecb80fc2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java index 1848628..e91b904 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java @@ -51,9 +51,11 @@ public class FunctionHolder { this.addBody = methods.get("add"); this.evalBody = methods.get("eval"); Preconditions.checkNotNull(evalBody); + Preconditions.checkArgument(!evalBody.isEmpty()); this.parameters = parameters; this.returnValue = returnValue; this.imports = imports; + } public List<String> getImports() { @@ -154,12 +156,21 @@ public class FunctionHolder { public boolean matches(FunctionCall call){ - if(!softCompare(call.getMajorType(), returnValue.type)) return false; - if(call.args.size() != parameters.length) return false; + if(!softCompare(call.getMajorType(), returnValue.type)){ +// logger.debug(String.format("Call [%s] didn't match as return type [%s] was different than expected [%s]. ", call.getDefinition().getName(), returnValue.type, call.getMajorType())); + return false; + } + if(call.args.size() != parameters.length){ +// logger.debug(String.format("Call [%s] didn't match as the number of arguments provided [%d] were different than expected [%d]. ", call.getDefinition().getName(), parameters.length, call.args.size())); + return false; + } for(int i =0; i < parameters.length; i++){ ValueReference param = parameters[i]; LogicalExpression arg = call.args.get(i); - if(!softCompare(param.type, arg.getMajorType())) return false; + if(!softCompare(param.type, arg.getMajorType())){ +// logger.debug(String.format("Call [%s] didn't match as the argument [%s] didn't match the expected type [%s]. ", call.getDefinition().getName(), arg.getMajorType(), param.type)); + return false; + } } return true; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecb80fc2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/MethodGrabbingVisitor.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/MethodGrabbingVisitor.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/MethodGrabbingVisitor.java index c96d6e0..75a6ff2 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/MethodGrabbingVisitor.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/MethodGrabbingVisitor.java @@ -6,6 +6,7 @@ import java.util.Map; import org.codehaus.janino.Java; import org.codehaus.janino.Java.ClassDeclaration; import org.codehaus.janino.Java.MethodDeclarator; +import org.codehaus.janino.UnparseVisitor; import org.codehaus.janino.util.Traverser; import com.google.common.collect.Maps; @@ -38,10 +39,15 @@ public class MethodGrabbingVisitor{ @Override public void traverseMethodDeclarator(MethodDeclarator md) { // logger.debug(c.getName() + ": Found {}, include {}", md.name, captureMethods); + if(captureMethods){ StringWriter writer = new StringWriter(); ModifiedUnparseVisitor v = new ModifiedUnparseVisitor(writer); +// UnparseVisitor v = new UnparseVisitor(writer); + md.accept(v); + v.close(); + writer.flush(); methods.put(md.name, writer.getBuffer().toString()); } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecb80fc2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ModifiedUnparseVisitor.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ModifiedUnparseVisitor.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ModifiedUnparseVisitor.java index 72b2008..be03bac 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ModifiedUnparseVisitor.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ModifiedUnparseVisitor.java @@ -189,17 +189,21 @@ public class ModifiedUnparseVisitor implements ComprehensiveVisitor { } } public void visitMethodDeclarator(Java.MethodDeclarator md) { - if (md.optionalStatements == null) { - this.pw.print(';'); - } else - if (md.optionalStatements.isEmpty()) { - this.pw.print(" "); - } else - { - this.pw.print(AutoIndentWriter.INDENT); - this.unparseStatements(md.optionalStatements); - this.pw.print(AutoIndentWriter.UNINDENT); - } + // modified to remove method declaration printing. + if (md.optionalStatements == null) { + this.pw.print(';'); + } else + if (md.optionalStatements.isEmpty()) { + this.pw.print(" {}"); + } else + { + this.pw.println(' '); + this.pw.print(AutoIndentWriter.INDENT); + this.unparseStatements(md.optionalStatements); + this.pw.print(AutoIndentWriter.UNINDENT); + this.pw.print(' '); + } + } public void visitFieldDeclaration(Java.FieldDeclaration fd) { this.unparseDocComment(fd); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecb80fc2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java index 6e3a4aa..11b9243 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java @@ -20,7 +20,6 @@ package org.apache.drill.exec.physical.config; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.ops.FragmentContext; @@ -30,10 +29,8 @@ import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.store.RecordReader; import org.apache.drill.exec.vector.AllocationHelper; -import org.apache.drill.exec.vector.FixedWidthVector; import org.apache.drill.exec.vector.TypeHelper; import org.apache.drill.exec.vector.ValueVector; -import org.apache.drill.exec.vector.VariableWidthVector; public class MockRecordReader implements RecordReader { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockRecordReader.class); @@ -63,7 +60,7 @@ public class MockRecordReader implements RecordReader { MaterializedField f = MaterializedField.create(new SchemaPath(name, ExpressionPosition.UNKNOWN), type); ValueVector v; v = TypeHelper.getNewVector(f, context.getAllocator()); - AllocationHelper.allocate(v, length, 50); + AllocationHelper.allocate(v, length, 50, 4); return v; @@ -95,7 +92,7 @@ public class MockRecordReader implements RecordReader { recordsRead += recordSetSize; for(ValueVector v : valueVectors){ - AllocationHelper.allocate(v, recordSetSize, 50); + AllocationHelper.allocate(v, recordSetSize, 50, 5); logger.debug("MockRecordReader: Generating random data for VV of type " + v.getClass().getName()); ValueVector.Mutator m = v.getMutator(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecb80fc2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java index 1631e70..69c17f4 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java @@ -4,10 +4,18 @@ public class AllocationHelper { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AllocationHelper.class); public static void allocate(ValueVector v, int valueCount, int bytesPerValue){ + allocate(v, valueCount, bytesPerValue, 5); + } + + public static void allocate(ValueVector v, int valueCount, int bytesPerValue, int repeatedPerTop){ if(v instanceof FixedWidthVector){ ((FixedWidthVector) v).allocateNew(valueCount); }else if(v instanceof VariableWidthVector){ ((VariableWidthVector) v).allocateNew(valueCount * bytesPerValue, valueCount); + }else if(v instanceof RepeatedFixedWidthVector){ + ((RepeatedFixedWidthVector) v).allocateNew(valueCount, valueCount * repeatedPerTop); + }else if(v instanceof RepeatedVariableWidthVector){ + ((RepeatedVariableWidthVector) v).allocateNew(valueCount * bytesPerValue, valueCount, valueCount * repeatedPerTop); }else{ throw new UnsupportedOperationException(); }
