DRILL-1671, DRILL-1653, DRILL-1652: Fixes for flatten bugs
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/60736dea Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/60736dea Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/60736dea Branch: refs/heads/master Commit: 60736dea0fc7099f3773616c1995df4789221cdb Parents: ed96249 Author: Jacques Nadeau <jacq...@apache.org> Authored: Mon Nov 10 22:16:19 2014 -0800 Committer: Jacques Nadeau <jacq...@apache.org> Committed: Tue Nov 11 16:48:45 2014 -0800 ---------------------------------------------------------------------- .../drill/common/expression/SchemaPath.java | 3 +- .../codegen/templates/RepeatedValueVectors.java | 30 +-- .../exec/planner/physical/DrillFlattenPrel.java | 75 ------- .../exec/planner/physical/FlattenPrel.java | 80 ++++++++ .../visitor/RewriteProjectToFlatten.java | 4 +- .../visitor/SplitUpComplexExpressions.java | 2 +- .../drill/exec/record/MaterializedField.java | 22 +- .../exec/vector/complex/RepeatedMapVector.java | 35 +++- .../exec/vector/complex/fn/JsonReader.java | 3 +- .../exec/physical/impl/flatten/TestFlatten.java | 51 +++++ .../src/test/resources/flatten/empty-rm.json | 7 + .../test/resources/flatten/many-arrays-50.json | 203 +++++++++++++++++++ .../src/test/resources/flatten/missing-map.json | 11 + .../flatten/single-user-transactions.json | 18 ++ 14 files changed, 445 insertions(+), 99 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java index 3b31a15..1d7f922 100644 --- a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java +++ b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java @@ -233,8 +233,7 @@ public class SchemaPath extends LogicalExpressionBase { @Override public String toString() { - String expr = ExpressionStringBuilder.toString(this); - return "SchemaPath ["+ expr + "]"; + return ExpressionStringBuilder.toString(this); } public String toExpr() { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java index 2853e83..0f86db9 100644 --- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java +++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java @@ -18,6 +18,7 @@ import java.lang.Override; +import org.apache.drill.exec.record.TransferPair; import org.apache.drill.exec.vector.RepeatedFixedWidthVector; import org.apache.drill.exec.vector.UInt4Vector; import org.mortbay.jetty.servlet.Holder; @@ -102,20 +103,25 @@ public final class Repeated${minor.class}Vector extends BaseValueVector implemen clear(); } - public void splitAndTransferTo(int startIndex, int length, Repeated${minor.class}Vector target) { - int startPos = offsets.getAccessor().get(startIndex); - int endPos = offsets.getAccessor().get(startIndex+length); - values.splitAndTransferTo(startIndex, endPos-startPos, target.values); - target.offsets.clear(); - target.offsets.allocateNew(endPos - startPos + 1); + public void splitAndTransferTo(final int startIndex, final int groups, Repeated${minor.class}Vector to) { + final UInt4Vector.Accessor a = offsets.getAccessor(); + final UInt4Vector.Mutator m = to.offsets.getMutator(); + + final int startPos = offsets.getAccessor().get(startIndex); + final int endPos = offsets.getAccessor().get(startIndex + groups); + final int valuesToCopy = endPos - startPos; + + values.splitAndTransferTo(startPos, valuesToCopy, to.values); + to.offsets.clear(); + to.offsets.allocateNew(valuesToCopy + 1); int normalizedPos = 0; - for (int i=0; i<length+1;i++) { - normalizedPos = offsets.getAccessor().get(startIndex+i) - startPos; - target.offsets.getMutator().set(i, normalizedPos); + for (int i=0; i < groups + 1;i++ ) { + normalizedPos = a.get(startIndex+i) - startPos; + m.set(i, normalizedPos); } - target.parentValueCount = length; - target.childValueCount = offsets.getAccessor().get(startIndex+length) - startPos; - target.offsets.getMutator().setValueCount(length); + to.parentValueCount = groups; + to.childValueCount = valuesToCopy; + m.setValueCount(groups); } private class TransferImpl implements TransferPair{ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillFlattenPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillFlattenPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillFlattenPrel.java deleted file mode 100644 index 1409347..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillFlattenPrel.java +++ /dev/null @@ -1,75 +0,0 @@ -/******************************************************************************* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ -package org.apache.drill.exec.planner.physical; - -import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.expression.PathSegment; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.exec.physical.base.PhysicalOperator; -import org.apache.drill.exec.physical.config.FlattenPOP; -import org.apache.drill.exec.planner.common.DrillFilterRelBase; -import org.apache.drill.exec.planner.logical.DrillOptiq; -import org.apache.drill.exec.planner.logical.DrillParseContext; -import org.apache.drill.exec.planner.physical.visitor.PrelVisitor; -import org.apache.drill.exec.record.BatchSchema; -import org.eigenbase.rel.RelNode; -import org.eigenbase.relopt.RelOptCluster; -import org.eigenbase.relopt.RelTraitSet; -import org.eigenbase.rex.RexNode; - -import java.io.IOException; -import java.util.Iterator; -import java.util.List; - -public class DrillFlattenPrel extends SinglePrel implements Prel { - - RexNode toFlatten; - - public DrillFlattenPrel(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode toFlatten) { - super(cluster, traits, child); - this.toFlatten = toFlatten; - } - - @Override - public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) { - return new DrillFlattenPrel(getCluster(), traitSet, sole(inputs), toFlatten); - } - - @Override - public Iterator<Prel> iterator() { - return PrelUtil.iter(getChild()); - } - - @Override - public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException { - Prel child = (Prel) this.getChild(); - - PhysicalOperator childPOP = child.getPhysicalOperator(creator); - FlattenPOP f = new FlattenPOP(childPOP, (SchemaPath) getFlattenExpression(new DrillParseContext())); - return creator.addMetadata(this, f); - } - - @Override - public BatchSchema.SelectionVectorMode getEncoding() { - return BatchSchema.SelectionVectorMode.NONE; - } - - protected LogicalExpression getFlattenExpression(DrillParseContext context){ - return DrillOptiq.toDrill(context, getChild(), toFlatten); - } -} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java new file mode 100644 index 0000000..18a2cc2 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java @@ -0,0 +1,80 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.exec.planner.physical; + +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.PathSegment; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.physical.base.PhysicalOperator; +import org.apache.drill.exec.physical.config.FlattenPOP; +import org.apache.drill.exec.planner.common.DrillFilterRelBase; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.physical.visitor.PrelVisitor; +import org.apache.drill.exec.record.BatchSchema; +import org.eigenbase.rel.RelNode; +import org.eigenbase.rel.RelWriter; +import org.eigenbase.relopt.RelOptCluster; +import org.eigenbase.relopt.RelTraitSet; +import org.eigenbase.rex.RexNode; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + +public class FlattenPrel extends SinglePrel implements Prel { + + RexNode toFlatten; + + public FlattenPrel(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode toFlatten) { + super(cluster, traits, child); + this.toFlatten = toFlatten; + } + + @Override + public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) { + return new FlattenPrel(getCluster(), traitSet, sole(inputs), toFlatten); + } + + @Override + public Iterator<Prel> iterator() { + return PrelUtil.iter(getChild()); + } + + @Override + public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException { + Prel child = (Prel) this.getChild(); + + PhysicalOperator childPOP = child.getPhysicalOperator(creator); + FlattenPOP f = new FlattenPOP(childPOP, (SchemaPath) getFlattenExpression(new DrillParseContext())); + return creator.addMetadata(this, f); + } + + public RelWriter explainTerms(RelWriter pw) { + return super.explainTerms(pw).item("flattenField", this.toFlatten); + } + + @Override + public BatchSchema.SelectionVectorMode getEncoding() { + return BatchSchema.SelectionVectorMode.NONE; + } + + protected LogicalExpression getFlattenExpression(DrillParseContext context){ + return DrillOptiq.toDrill(context, getChild(), toFlatten); + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java index d4b3573..1055411 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java @@ -21,7 +21,7 @@ import com.google.common.collect.Lists; import net.hydromatic.optiq.tools.RelConversionException; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; -import org.apache.drill.exec.planner.physical.DrillFlattenPrel; +import org.apache.drill.exec.planner.physical.FlattenPrel; import org.apache.drill.exec.planner.physical.Prel; import org.apache.drill.exec.planner.physical.ProjectPrel; import org.apache.drill.exec.planner.physical.visitor.BasePrelVisitor; @@ -103,7 +103,7 @@ public class RewriteProjectToFlatten extends BasePrelVisitor<Prel, Object, RelCo // TODO - figure out what is the right setting for the traits Prel newChild = ((Prel)project.getInput(0)).accept(this, null); ProjectPrel newProject = new ProjectPrel(node.getCluster(), project.getTraitSet(), newChild, exprList, new RelRecordType(relDataTypes)); - DrillFlattenPrel flatten = new DrillFlattenPrel(project.getCluster(), project.getTraitSet(), newProject, flatttenExpr); + FlattenPrel flatten = new FlattenPrel(project.getCluster(), project.getTraitSet(), newProject, flatttenExpr); return flatten; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java index 0a49f3a..89c8447 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java @@ -26,7 +26,7 @@ import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.planner.logical.DrillRel; import org.apache.drill.exec.planner.logical.DrillScanRel; import org.apache.drill.exec.planner.logical.DrillTable; -import org.apache.drill.exec.planner.physical.DrillFlattenPrel; +import org.apache.drill.exec.planner.physical.FlattenPrel; import org.apache.drill.exec.planner.physical.Prel; import org.apache.drill.exec.planner.physical.PrelUtil; import org.apache.drill.exec.planner.physical.ProjectPrel; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java index b4da2b4..7f7dcfa 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java @@ -18,10 +18,12 @@ package org.apache.drill.exec.record; import java.util.Collection; +import java.util.Iterator; import java.util.Map; import java.util.Set; import com.google.common.collect.Sets; + import org.apache.drill.common.expression.FieldReference; import org.apache.drill.common.expression.PathSegment; import org.apache.drill.common.expression.SchemaPath; @@ -215,9 +217,27 @@ public class MaterializedField { return true; } + @Override public String toString() { - return "MaterializedField [path=" + key.path + ", type=" + Types.toString(key.type) + "]"; + final int maxLen = 10; + String childStr = children != null && !children.isEmpty() ? toString(children, maxLen) : ""; + return key.path + "(" + key.type.getMinorType().name() + ":" + key.type.getMode().name() + ")" + childStr; + } + + + private String toString(Collection<?> collection, int maxLen) { + StringBuilder builder = new StringBuilder(); + builder.append("["); + int i = 0; + for (Iterator<?> iterator = collection.iterator(); iterator.hasNext() && i < maxLen; i++) { + if (i > 0){ + builder.append(", "); + } + builder.append(iterator.next()); + } + builder.append("]"); + return builder.toString(); } public Key key() { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java index 9b7011c..7ddac23 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java @@ -360,6 +360,9 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat public boolean copyValueSafe(int srcIndex, int destIndex) { RepeatedMapHolder holder = new RepeatedMapHolder(); accessor.get(srcIndex, holder); + if(destIndex >= to.getValueCapacity()){ + return false; + } to.populateEmpties(destIndex+1); int newIndex = to.offsets.getAccessor().get(destIndex); //todo: make these bulk copies @@ -377,8 +380,30 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat } @Override - public void splitAndTransfer(int startIndex, int length) { - throw new UnsupportedOperationException(); + public void splitAndTransfer(final int groupStart, final int groups) { + final UInt4Vector.Accessor a = offsets.getAccessor(); + final UInt4Vector.Mutator m = to.offsets.getMutator(); + + final int startPos = a.get(groupStart); + final int endPos = a.get(groupStart+groups); + final int valuesToCopy = endPos - startPos; + + to.offsets.clear(); + to.offsets.allocateNew(groups + 1); + int normalizedPos = 0; + + for (int i=0; i < groups+1; i++) { + normalizedPos = a.get(groupStart+i) - startPos; + m.set(i, normalizedPos); + } + + m.setValueCount(groups + 1); + to.lastSet = groups; + + for (TransferPair p : pairs) { + p.splitAndTransfer(startPos, valuesToCopy); + } + } } @@ -444,7 +469,9 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat SerializedField.Builder b = getField() // .getAsBuilder() // .setBufferLength(getBufferSize()) // - .setGroupCount(accessor.getGroupCount()); + .setGroupCount(accessor.getGroupCount()) + // while we don't need to actually read this on load, we need it to make sure we don't skip deserialization of this vector + .setValueCount(accessor.getValueCount()); for (ValueVector v : vectors.values()) { b.addChild(v.getMetadata()); } @@ -503,7 +530,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat } public void get(int index, RepeatedMapHolder holder) { - assert index < getValueCapacity()-1; + assert index < getValueCapacity() : String.format("Attempted to access index %d when value capacity is %d", index, getValueCapacity()); holder.start = offsets.getAccessor().get(index); holder.end = offsets.getAccessor().get(index+1); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java index 17e266f..0ca24e7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java @@ -197,7 +197,6 @@ private boolean writeToVector(ComplexWriter writer, JsonToken t) throws JsonPars map.start(); outside: while(true) { if (!map.ok()) { - logger.warn("Error reported. Quit writing"); break; } JsonToken t = parser.nextToken(); @@ -320,7 +319,7 @@ private boolean writeToVector(ComplexWriter writer, JsonToken t) throws JsonPars list.start(); outside: while (true) { if (!list.ok()) { - logger.warn("Error reported. Quit writing"); +// logger.warn("Error reported. Quit writing"); break; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java index d4c19a3..960da65 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java @@ -17,7 +17,10 @@ ******************************************************************************/ package org.apache.drill.exec.physical.impl.flatten; +import static org.junit.Assert.assertEquals; + import org.apache.drill.BaseTestQuery; +import org.junit.Ignore; import org.junit.Test; public class TestFlatten extends BaseTestQuery { @@ -27,6 +30,7 @@ public class TestFlatten extends BaseTestQuery { * - /tmp/yelp_academic_dataset_business.json * - /tmp/mapkv.json * - /tmp/drill1665.json + * - /tmp/bigfile.json */ public static boolean RUN_ADVANCED_TESTS = false; @@ -38,6 +42,53 @@ public class TestFlatten extends BaseTestQuery { } @Test + public void drill1671() throws Exception{ + int rowCount = testSql("select * from (select count(*) as cnt from (select id, flatten(evnts1), flatten(evnts2), flatten(evnts3), flatten(evnts4), flatten(evnts5), flatten(evnts6), flatten(evnts7), flatten(evnts8), flatten(evnts9), flatten(evnts10), flatten(evnts11) from cp.`/flatten/many-arrays-50.json`)x )y where cnt = 2048"); + assertEquals(rowCount, 1); + } + + @Test + @Ignore("not yet fixed") + public void drill1660() throws Exception { + test("select * from cp.`/flatten/empty-rm.json`"); + } + + @Test + public void drill1653() throws Exception{ + int rowCount = testSql("select * from (select sum(t.flat.`value`) as sm from (select id, flatten(kvgen(m)) as flat from cp.`/flatten/missing-map.json`)t) where sm = 10 "); + assertEquals(rowCount, 1); + } + + @Test + public void drill1652() throws Exception { + if(RUN_ADVANCED_TESTS){ + test("select uid, flatten(transactions) from dfs.`/tmp/bigfile.json`"); + } + } + + @Test + @Ignore("Still not working.") + public void drill1649() throws Exception { + test("select event_info.uid, transaction_info.trans_id, event_info.event.evnt_id\n" + + "from (\n" + + " select userinfo.transaction.trans_id trans_id, max(userinfo.event.event_time) max_event_time\n" + + " from (\n" + + " select uid, flatten(events) event, flatten(transactions) transaction from cp.`/flatten/single-user-transactions.json`\n" + + " ) userinfo\n" + + " where userinfo.transaction.trans_time >= userinfo.event.event_time\n" + + " group by userinfo.transaction.trans_id\n" + + ") transaction_info\n" + + "inner join\n" + + "(\n" + + " select uid, flatten(events) event\n" + + " from cp.`/flatten/single-user-transactions.json`\n" + + ") event_info\n" + + "on transaction_info.max_event_time = event_info.event.event_time;"); + } + + + + @Test public void testKVGenFlatten1() throws Exception { // works - TODO and verify results test("select flatten(kvgen(f1)) as monkey, x " + http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/resources/flatten/empty-rm.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/flatten/empty-rm.json b/exec/java-exec/src/test/resources/flatten/empty-rm.json new file mode 100644 index 0000000..7927a3f --- /dev/null +++ b/exec/java-exec/src/test/resources/flatten/empty-rm.json @@ -0,0 +1,7 @@ +{ + "id":1, + "evnts":[ + {}, + {"key":null} + ] +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/resources/flatten/many-arrays-50.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/flatten/many-arrays-50.json b/exec/java-exec/src/test/resources/flatten/many-arrays-50.json new file mode 100644 index 0000000..c25565e --- /dev/null +++ b/exec/java-exec/src/test/resources/flatten/many-arrays-50.json @@ -0,0 +1,203 @@ +{ + "id":1, + "evnts1":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts2":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts3":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts4":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts5":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts6":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts7":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts8":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts9":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts10":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts11":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts12":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts13":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts14":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts15":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts16":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts17":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts18":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts19":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts20":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts21":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts22":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts23":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts24":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts25":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts26":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts27":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts28":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts29":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts30":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts31":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts32":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts33":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts34":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts35":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts36":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts37":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts38":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts39":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts40":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts41":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts42":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts43":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts44":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts45":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts46":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts47":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts48":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts49":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ], + "evnts50":[ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000} + ] +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/resources/flatten/missing-map.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/flatten/missing-map.json b/exec/java-exec/src/test/resources/flatten/missing-map.json new file mode 100644 index 0000000..5d8f815 --- /dev/null +++ b/exec/java-exec/src/test/resources/flatten/missing-map.json @@ -0,0 +1,11 @@ +{ + "id": 1, + "m": {"a":1,"b":2} +} +{ + "id": 2 +} +{ + "id": 3, + "m": {"c":3,"d":4} +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/resources/flatten/single-user-transactions.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/flatten/single-user-transactions.json b/exec/java-exec/src/test/resources/flatten/single-user-transactions.json new file mode 100644 index 0000000..cff5532 --- /dev/null +++ b/exec/java-exec/src/test/resources/flatten/single-user-transactions.json @@ -0,0 +1,18 @@ +{ + "uid" : 1, + "events" : [ + { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}, + { "evnt_id":"e2", "campaign_id":"c1", "event_name":"e2_name", "event_time":2000000}, + { "evnt_id":"e3", "campaign_id":"c1", "event_name":"e3_name", "event_time":3000000}, + { "evnt_id":"e4", "campaign_id":"c1", "event_name":"e4_name", "event_time":4000000}, + { "evnt_id":"e5", "campaign_id":"c2", "event_name":"e5_name", "event_time":5000000}, + { "evnt_id":"e6", "campaign_id":"c1", "event_name":"e6_name", "event_time":6000000}, + { "evnt_id":"e7", "campaign_id":"c1", "event_name":"e7_name", "event_time":7000000}, + { "evnt_id":"e8", "campaign_id":"c2", "event_name":"e8_name", "event_time":8000000}, + { "evnt_id":"e9", "campaign_id":"c2", "event_name":"e9_name", "event_time":9000000} + ], + "transactions" : [ + { "trans_id":"t1", "amount":100, "trans_time":7777777}, + { "trans_id":"t2", "amount":100, "trans_time":8888888} + ] +}
