DRILL-190 (part3) Fixes for integration between JSON changes, updated CodeGenerator and merge join
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/5232b0e1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/5232b0e1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/5232b0e1 Branch: refs/heads/master Commit: 5232b0e1423deecb206e79d5978e4fe7db8197b6 Parents: e0bac2f Author: Jacques Nadeau <jacq...@apache.org> Authored: Wed Aug 28 17:48:36 2013 -0700 Committer: Jacques Nadeau <jacq...@apache.org> Committed: Wed Aug 28 20:37:40 2013 -0700 ---------------------------------------------------------------------- .../drill/exec/physical/impl/ScanBatch.java | 2 +- .../physical/impl/aggregate/InternalBatch.java | 2 +- .../exec/physical/impl/join/JoinEvaluator.java | 10 ------ .../physical/impl/join/JoinInnerSignature.java | 35 -------------------- .../exec/physical/impl/join/MergeJoinBatch.java | 4 +-- .../partitionsender/OutgoingRecordBatch.java | 2 +- .../drill/exec/record/AbstractRecordBatch.java | 2 +- .../drill/exec/record/RecordBatchLoader.java | 2 +- .../drill/exec/record/VectorContainer.java | 2 +- .../exec/physical/impl/join/TestMergeJoin.java | 3 +- .../test/resources/join/merge_single_batch.json | 20 +++++++---- 11 files changed, 24 insertions(+), 60 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java index 192c03c..ae043ec 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java @@ -134,7 +134,7 @@ public class ScanBatch implements RecordBatch { @Override public VectorWrapper<?> getValueAccessorById(int fieldId, Class<?> clazz) { - return container.getVectorAccessor(fieldId, clazz); + return container.getValueAccessorById(fieldId, clazz); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/InternalBatch.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/InternalBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/InternalBatch.java index 343dbe5..77dd682 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/InternalBatch.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/InternalBatch.java @@ -59,7 +59,7 @@ public class InternalBatch implements Iterable<VectorWrapper<?>>{ } public VectorWrapper<?> getValueAccessorById(int fieldId, Class<?> clazz){ - return container.getVectorAccessor(fieldId, clazz); + return container.getValueAccessorById(fieldId, clazz); } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinEvaluator.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinEvaluator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinEvaluator.java deleted file mode 100644 index beb3e28..0000000 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinEvaluator.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.apache.drill.exec.physical.impl.join; - -import org.apache.drill.exec.exception.SchemaChangeException; -import org.apache.drill.exec.ops.FragmentContext; -import org.apache.drill.exec.record.VectorContainer; - -public interface JoinEvaluator { - public abstract void doSetup(FragmentContext context, JoinStatus status, VectorContainer outgoing) throws SchemaChangeException; - -} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinInnerSignature.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinInnerSignature.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinInnerSignature.java deleted file mode 100644 index 1081244..0000000 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinInnerSignature.java +++ /dev/null @@ -1,35 +0,0 @@ -/******************************************************************************* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package org.apache.drill.exec.physical.impl.join; - -import static org.apache.drill.exec.compile.sig.GeneratorMapping.GM; - -import javax.inject.Named; - -import org.apache.drill.exec.compile.sig.CodeGeneratorSignature; -import org.apache.drill.exec.compile.sig.GeneratorMapping; -import org.apache.drill.exec.compile.sig.MappingSet; -import org.apache.drill.exec.exception.SchemaChangeException; -import org.apache.drill.exec.ops.FragmentContext; -import org.apache.drill.exec.record.VectorContainer; - - -public interface JoinInnerSignature extends CodeGeneratorSignature { - -} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java index a2b84da..af33ca4 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java @@ -317,7 +317,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> { JVar vvIn = cg.declareVectorValueSetupAndMember("incomingLeft", new TypedFieldId(vw.getField().getType(), vectorId)); JVar vvOut = cg.declareVectorValueSetupAndMember("outgoing", - new TypedFieldId(vw.getField().getType(),vectorId, true)); + new TypedFieldId(vw.getField().getType(),vectorId)); // todo: check for room in vvOut cg.getEvalBlock().add(vvOut.invoke("copyFrom") .arg(COPY_LEFT_MAPPING.getValueReadIndex()) @@ -336,7 +336,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> { JVar vvIn = cg.declareVectorValueSetupAndMember("incomingRight", new TypedFieldId(vw.getField().getType(), vectorId - rightVectorBase)); JVar vvOut = cg.declareVectorValueSetupAndMember("outgoing", - new TypedFieldId(vw.getField().getType(),vectorId, true)); + new TypedFieldId(vw.getField().getType(),vectorId)); cg.getEvalBlock().add(vvOut.invoke("copyFrom") .arg(COPY_RIGHT_MAPPING.getValueReadIndex()) .arg(COPY_RIGHT_MAPPING.getValueWriteIndex()) http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/OutgoingRecordBatch.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/OutgoingRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/OutgoingRecordBatch.java index 0cefc52..e429402 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/OutgoingRecordBatch.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/OutgoingRecordBatch.java @@ -222,7 +222,7 @@ public class OutgoingRecordBatch implements RecordBatch { @Override public VectorWrapper<?> getValueAccessorById(int fieldId, Class<?> clazz) { - return vectorContainer.getVectorAccessor(fieldId, clazz); + return vectorContainer.getValueAccessorById(fieldId, clazz); } @Override http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java index a2584b8..ccd2468 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java @@ -65,7 +65,7 @@ public abstract class AbstractRecordBatch<T extends PhysicalOperator> implements @Override public VectorWrapper<?> getValueAccessorById(int fieldId, Class<?> clazz) { - return container.getVectorAccessor(fieldId, clazz); + return container.getValueAccessorById(fieldId, clazz); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java index 4d47404..9ac53f0 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java @@ -140,7 +140,7 @@ public class RecordBatchLoader implements Iterable<VectorWrapper<?>>{ } public VectorWrapper<?> getValueAccessorById(int fieldId, Class<?> clazz){ - return container.getVectorAccessor(fieldId, clazz); + return container.getValueAccessorById(fieldId, clazz); } public WritableBatch getWritableBatch(){ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java index 25036fc..e6c8bab 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java @@ -108,7 +108,7 @@ public class VectorContainer implements Iterable<VectorWrapper<?>> { } @SuppressWarnings("unchecked") - public <T extends ValueVector> VectorWrapper<T> getVectorAccessor(int fieldId, Class<?> clazz) { + public <T extends ValueVector> VectorWrapper<T> getValueAccessorById(int fieldId, Class<?> clazz) { VectorWrapper<?> va = wrappers.get(fieldId); assert va != null; if (va.getVectorClass() != clazz) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java index 38b8225..6aa651b 100644 --- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java +++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java @@ -112,9 +112,10 @@ public class TestMergeJoin { new NonStrictExpectations(){{ bitContext.getMetrics(); result = new MetricRegistry("test"); bitContext.getAllocator(); result = BufferAllocator.getAllocator(c); + bitContext.getConfig(); result = c; }}; - PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StorageEngineRegistry(bitContext)); PhysicalPlan plan = reader.readPhysicalPlan( Files.toString( FileUtils.getResourceAsFile("/join/merge_single_batch.json"), Charsets.UTF_8) http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_single_batch.json ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_single_batch.json b/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_single_batch.json index 0e4f79d..ad33d26 100644 --- a/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_single_batch.json +++ b/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_single_batch.json @@ -10,16 +10,24 @@ { @id:1, pop:"json-sub-scan", - entries:[ - {url: "#{LEFT_FILE}"} - ] + readEntries:[ + {path: "#{LEFT_FILE}"} + ], + engineConfig:{ + "type":"json", + "dfsName" : "file:///" + } }, { @id:2, pop:"json-sub-scan", - entries:[ - {url: "#{RIGHT_FILE}"} - ] + readEntries:[ + {path: "#{RIGHT_FILE}"} + ], + engineConfig:{ + "type":"json", + "dfsName" : "file:///" + } }, { @id: 3,
