Fix value vectors for e2e scan json pop
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/a15f5b19 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/a15f5b19 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/a15f5b19 Branch: refs/heads/master Commit: a15f5b19c7362793590d670ba1b932f6f8dfda69 Parents: 38ab96f Author: Timothy Chen <[email protected]> Authored: Sat Aug 10 01:32:13 2013 -0700 Committer: Timothy Chen <[email protected]> Committed: Sat Aug 10 01:32:13 2013 -0700 ---------------------------------------------------------------------- .../templates/NullableValueVectors.java | 5 +++++ .../templates/RepeatedValueVectors.java | 12 +++++----- .../apache/drill/exec/store/VectorHolder.java | 8 ++++++- .../drill/exec/vector/RepeatedMutator.java | 23 ++++++++++++++++++++ .../physical/impl/TestSimpleFragmentRun.java | 7 ++++-- .../resources/physical_json_scan_test1.json | 2 +- 6 files changed, 46 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a15f5b19/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java index ca222df..976c984 100644 --- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java +++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java @@ -58,7 +58,12 @@ public final class ${className} extends BaseValueVector implements <#if type.maj @Override public ByteBuf[] getBuffers() { + <#if type.major == "VarLen"> + ByteBuf[] valueBuffers = values.getBuffers(); + return new ByteBuf[]{bits.data, valueBuffers[0], valueBuffers[1]}; + <#else> return new ByteBuf[]{bits.data, values.data}; + </#if> } @Override http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a15f5b19/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java index f4a7049..5def096 100644 --- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java +++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java @@ -249,12 +249,16 @@ import com.google.common.collect.Lists; } } - public final class Mutator implements ValueVector.Mutator { + public final class Mutator implements RepeatedMutator { private Mutator(){ } + public void startNewGroup(int index) { + offsets.getMutator().set(index+1, offsets.getAccessor().get(index)); + } + /** * Add an element to the given record index. This is similar to the set() method in other * value vectors, except that it permits setting multiple values for a single record. @@ -264,18 +268,12 @@ import com.google.common.collect.Lists; */ public void add(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) { int nextOffset = offsets.getAccessor().get(index+1); - if (index > 0 && nextOffset == 0) { - nextOffset = offsets.getAccessor().get(index); - } values.getMutator().set(nextOffset, value); offsets.getMutator().set(index+1, nextOffset+1); } public void add(int index, ${minor.class}Holder holder){ int nextOffset = offsets.getAccessor().get(index+1); - if (index > 0 && nextOffset == 0) { - nextOffset = offsets.getAccessor().get(index); - } values.getMutator().set(nextOffset, holder); offsets.getMutator().set(index+1, nextOffset+1); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a15f5b19/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java index 2c28082..be0bea8 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java @@ -45,7 +45,13 @@ public class VectorHolder { } public void setGroupCount(int groupCount) { - this.groupCount = groupCount; + if(this.groupCount < groupCount) { + RepeatedMutator mutator = (RepeatedMutator) vector.getMutator(); + while(this.groupCount < groupCount) { + mutator.startNewGroup(this.groupCount + 1); + this.groupCount++; + } + } } public boolean hasEnoughSpace(int newLength) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a15f5b19/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedMutator.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedMutator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedMutator.java new file mode 100644 index 0000000..3e81d75 --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedMutator.java @@ -0,0 +1,23 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package org.apache.drill.exec.vector; + +public interface RepeatedMutator extends ValueVector.Mutator { + public void startNewGroup(int index); +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a15f5b19/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java index cabe9b3..db6c437 100644 --- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java +++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java @@ -116,7 +116,10 @@ public class TestSimpleFragmentRun extends PopUnitTestBase { // run query. bit.run(); client.connect(); - List<QueryResultBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/physical_json_scan_test1.json"), Charsets.UTF_8)); + List<QueryResultBatch> results = client.runQuery(QueryType.PHYSICAL, + Files.toString(FileUtils.getResourceAsFile("/physical_json_scan_test1.json"), Charsets.UTF_8) + .replace("#{TEST_FILE}", FileUtils.getResourceAsFile("/scan_json_test_1.json").toURI().toString()) + ); // look at records RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator()); @@ -156,7 +159,7 @@ public class TestSimpleFragmentRun extends PopUnitTestBase { System.out.println(); - for (int r = 0; i < batchLoader.getRecordCount(); r++) { + for (int r = 0; r < batchLoader.getRecordCount(); r++) { boolean first = true; recordCount++; for (VectorWrapper<?> v : batchLoader) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a15f5b19/sandbox/prototype/exec/java-exec/src/test/resources/physical_json_scan_test1.json ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/physical_json_scan_test1.json b/sandbox/prototype/exec/java-exec/src/test/resources/physical_json_scan_test1.json index 91eb80c..6f08937 100644 --- a/sandbox/prototype/exec/java-exec/src/test/resources/physical_json_scan_test1.json +++ b/sandbox/prototype/exec/java-exec/src/test/resources/physical_json_scan_test1.json @@ -11,7 +11,7 @@ @id:1, pop:"json-scan", entries:[ - {url: "file:////home/tnachen/src/incubator-drill/sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_1.json"} + {url: "#{TEST_FILE}"} ] }, {
