Fix getBuffers and address comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/6f9dadb9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/6f9dadb9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/6f9dadb9 Branch: refs/heads/master Commit: 6f9dadb952db3740cb187057967785d0ffca8e3e Parents: b1e48b3 Author: Timothy Chen <[email protected]> Authored: Tue Aug 13 20:42:53 2013 -0700 Committer: Timothy Chen <[email protected]> Committed: Tue Aug 13 20:42:53 2013 -0700 ---------------------------------------------------------------------- .../templates/NullableValueVectors.java | 8 ++------ .../templates/RepeatedValueVectors.java | 3 ++- .../org/apache/drill/exec/schema/DiffSchema.java | 13 ++++++------- .../apache/drill/exec/store/JSONRecordReader.java | 16 ++++++++-------- .../org/apache/drill/exec/store/VectorHolder.java | 12 ++++++------ 5 files changed, 24 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f9dadb9/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java index 976c984..483166b 100644 --- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java +++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java @@ -17,6 +17,7 @@ import io.netty.buffer.ByteBuf; import java.io.Closeable; import java.util.List; +import org.apache.commons.lang3.ArrayUtils; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.proto.SchemaDefProtos; import org.apache.drill.exec.proto.UserBitShared.FieldMetadata; @@ -58,12 +59,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj @Override public ByteBuf[] getBuffers() { - <#if type.major == "VarLen"> - ByteBuf[] valueBuffers = values.getBuffers(); - return new ByteBuf[]{bits.data, valueBuffers[0], valueBuffers[1]}; - <#else> - return new ByteBuf[]{bits.data, values.data}; - </#if> + return ArrayUtils.addAll(bits.getBuffers(), values.getBuffers()); } @Override http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f9dadb9/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java index 5def096..c1660e8 100644 --- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java +++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java @@ -17,6 +17,7 @@ import java.io.Closeable; import java.util.Random; import java.util.Vector; +import org.apache.commons.lang3.ArrayUtils; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.proto.SchemaDefProtos; import org.apache.drill.exec.proto.UserBitShared.FieldMetadata; @@ -176,7 +177,7 @@ import com.google.common.collect.Lists; </#if> public ByteBuf[] getBuffers() { - return new ByteBuf[]{offsets.data, values.data}; + return ArrayUtils.addAll(offsets.getBuffers(), values.getBuffers()); } public void clear(){ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f9dadb9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/DiffSchema.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/DiffSchema.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/DiffSchema.java index 68c3e12..81a396e 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/DiffSchema.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/DiffSchema.java @@ -19,14 +19,13 @@ package org.apache.drill.exec.schema; import com.google.common.collect.Lists; -import org.apache.drill.exec.physical.impl.OutputMutator; import java.util.List; public class DiffSchema { List<Field> addedFields; List<Field> removedFields; - boolean hasChanged = false; + boolean changed = false; public DiffSchema() { this.addedFields = Lists.newArrayList(); @@ -35,22 +34,22 @@ public class DiffSchema { public void recordNewField(Field field) { addedFields.add(field); - hasChanged = true; + changed = true; } public void reset() { addedFields.clear(); removedFields.clear(); - hasChanged = false; + changed = false; } public void addRemovedField(Field field) { removedFields.add(field); - hasChanged = true; + changed = true; } - public boolean isHasChanged() { - return hasChanged; + public boolean isChanged() { + return changed; } @Override http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f9dadb9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java index 21b8c1b..8a2de63 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java @@ -125,7 +125,7 @@ public class JSONRecordReader implements RecordReader { outputMutator.removeField(field.getAsMaterializedField()); } - if (diffSchema.isHasChanged()) { + if (diffSchema.isChanged()) { outputMutator.setNewSchema(); } @@ -358,7 +358,7 @@ public class JSONRecordReader implements RecordReader { private static <T> boolean addValueToVector(int index, VectorHolder holder, T val, MinorType minorType, int groupCount) { switch (minorType) { case INT: { - holder.incAndCheckLength(32 + 1); + holder.incAndCheckLength(NullableIntHolder.WIDTH * 8 + 1); if (groupCount == 0) { if (val != null) { NullableIntVector int4 = (NullableIntVector) holder.getValueVector(); @@ -376,10 +376,10 @@ public class JSONRecordReader implements RecordReader { m.add(index, (Integer) val); } - return holder.hasEnoughSpace(32 + 1); + return holder.hasEnoughSpace(NullableIntHolder.WIDTH * 8 + 1); } case FLOAT4: { - holder.incAndCheckLength(32 + 1); + holder.incAndCheckLength(NullableFloat4Holder.WIDTH * 8 + 1); if (groupCount == 0) { if (val != null) { NullableFloat4Vector float4 = (NullableFloat4Vector) holder.getValueVector(); @@ -396,7 +396,7 @@ public class JSONRecordReader implements RecordReader { holder.setGroupCount(index); m.add(index, (Float) val); } - return holder.hasEnoughSpace(32 + 1); + return holder.hasEnoughSpace(NullableFloat4Holder.WIDTH * 8 + 1); } case VARCHAR: { if (val == null) { @@ -419,7 +419,7 @@ public class JSONRecordReader implements RecordReader { } } case BIT: { - holder.incAndCheckLength(1 + 1); + holder.incAndCheckLength(NullableBitHolder.WIDTH + 1); if (groupCount == 0) { if (val != null) { NullableBitVector bit = (NullableBitVector) holder.getValueVector(); @@ -436,7 +436,7 @@ public class JSONRecordReader implements RecordReader { holder.setGroupCount(index); m.add(index, (Boolean) val ? 1 : 0); } - return holder.hasEnoughSpace(1 + 1); + return holder.hasEnoughSpace(NullableBitHolder.WIDTH + 1); } default: throw new DrillRuntimeException("Type not supported to add value. Type: " + minorType); @@ -476,7 +476,7 @@ public class JSONRecordReader implements RecordReader { ValueVector v = TypeHelper.getNewVector(f, allocator); AllocationHelper.allocate(v, batchSize, 50); - holder = new VectorHolder(batchSize, v); + holder = new VectorHolder(v); valueVectorMap.put(fullFieldName, holder); outputMutator.addField(v); return holder; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f9dadb9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java index be0bea8..d2ad72a 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java @@ -27,8 +27,8 @@ public class VectorHolder { private ValueVector vector; private int currentLength; - VectorHolder(int length, ValueVector vector) { - this.length = length; + VectorHolder(ValueVector vector) { + this.length = vector.getValueCapacity(); this.vector = vector; } @@ -38,18 +38,18 @@ public class VectorHolder { public void incAndCheckLength(int newLength) { if (!hasEnoughSpace(newLength)) { - throw new BatchExceededException(length, currentLength + newLength); + throw new BatchExceededException(length, vector.getBufferSize() + newLength); } - count += 1; + currentLength += newLength; + count += 1; } public void setGroupCount(int groupCount) { if(this.groupCount < groupCount) { RepeatedMutator mutator = (RepeatedMutator) vector.getMutator(); while(this.groupCount < groupCount) { - mutator.startNewGroup(this.groupCount + 1); - this.groupCount++; + mutator.startNewGroup(++this.groupCount); } } }
