Repository: incubator-drill Updated Branches: refs/heads/DRILL-1459 [created] 5b61b573e
DRILL-1459: fix an issue that causes jvm to crash: fix field materialization to include child nodes during materialization; prevent RecordBatchLoader from creating multiple vectors for the same field; fix a test case; Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/5b61b573 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/5b61b573 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/5b61b573 Branch: refs/heads/DRILL-1459 Commit: 5b61b573e5c3c088049b3d3e22327889b8e5f257 Parents: 63d3008 Author: Hanifi Gunes <hgu...@maprtech.com> Authored: Thu Oct 2 20:44:04 2014 -0700 Committer: Parth Chandra <pchan...@maprtech.com> Committed: Mon Oct 27 10:06:42 2014 -0700 ---------------------------------------------------------------------- .../apache/drill/exec/record/MaterializedField.java | 16 +++++++++++----- .../apache/drill/exec/vector/complex/MapVector.java | 2 +- .../exec/vector/complex/writer/TestJsonReader.java | 3 --- 3 files changed, 12 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5b61b573/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java index 0ed74fb..b4da2b4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java @@ -17,9 +17,11 @@ */ package org.apache.drill.exec.record; -import java.util.List; +import java.util.Collection; import java.util.Map; +import java.util.Set; +import com.google.common.collect.Sets; import org.apache.drill.common.expression.FieldReference; import org.apache.drill.common.expression.PathSegment; import org.apache.drill.common.expression.SchemaPath; @@ -29,11 +31,11 @@ import org.apache.drill.common.types.Types; import org.apache.drill.exec.expr.TypeHelper; import org.apache.drill.exec.proto.UserBitShared.SerializedField; -import com.google.common.collect.Lists; public class MaterializedField { private Key key; - private List<MaterializedField> children = Lists.newArrayList(); + // use an ordered set as existing code relies on order (e,g. parquet writer) + private Set<MaterializedField> children = Sets.newLinkedHashSet(); private MaterializedField(SchemaPath path, MajorType type) { super(); @@ -41,7 +43,11 @@ public class MaterializedField { } public static MaterializedField create(SerializedField serField){ - return new MaterializedField(SchemaPath.create(serField.getNamePart()), serField.getMajorType()); + MaterializedField field = new MaterializedField(SchemaPath.create(serField.getNamePart()), serField.getMajorType()); + for (SerializedField sf:serField.getChildList()) { + field.addChild(MaterializedField.create(sf)); + } + return field; } public SerializedField.Builder getAsBuilder(){ @@ -50,7 +56,7 @@ public class MaterializedField { .setNamePart(key.path.getAsNamePart()); } - public List<MaterializedField> getChildren() { + public Collection<MaterializedField> getChildren() { return children; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5b61b573/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java index 3403fc3..1e4c8c4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java @@ -327,6 +327,7 @@ public class MapVector extends AbstractContainerVector { if (v == null) { // if we arrive here, we didn't have a matching vector. v = TypeHelper.getNewVector(fieldDef, allocator); + put(fieldDef.getLastName(), v); } if (fmd.getValueCount() == 0) { v.clear(); @@ -334,7 +335,6 @@ public class MapVector extends AbstractContainerVector { v.load(fmd, buf.slice(bufOffset, fmd.getBufferLength())); } bufOffset += fmd.getBufferLength(); - put(fieldDef.getLastName(), v); } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5b61b573/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java index e8f4331..f207bba 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java @@ -227,9 +227,6 @@ public class TestJsonReader extends BaseTestQuery { } private void testExistentColumns(RecordBatchLoader batchLoader, QueryResultBatch batch) throws SchemaChangeException { - - assertTrue(batchLoader.load(batch.getHeader().getDef(), batch.getData())); - VectorWrapper<?> vw = batchLoader.getValueAccessorById( RepeatedBigIntVector.class, // batchLoader.getValueVectorId(SchemaPath.getCompoundPath("field_1")).getFieldIds() //
