Repository: drill Updated Branches: refs/heads/master 288c95223 -> 68c933c75
DRILL-2997: Remove references to groupCount from SerializedField Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/68c933c7 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/68c933c7 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/68c933c7 Branch: refs/heads/master Commit: 68c933c75f3832abd5a56d31937c45d3087cee47 Parents: 288c952 Author: Hanifi Gunes <[email protected]> Authored: Mon Jun 8 16:47:45 2015 -0700 Committer: Hanifi Gunes <[email protected]> Committed: Mon Jun 15 16:14:52 2015 -0700 ---------------------------------------------------------------------- .../codegen/templates/RepeatedValueVectors.java | 35 +-- .../templates/VariableLengthVectors.java | 5 +- .../drill/exec/record/RecordBatchLoader.java | 2 +- .../drill/exec/vector/BaseValueVector.java | 2 +- .../exec/vector/complex/AbstractMapVector.java | 8 +- .../vector/complex/BaseRepeatedValueVector.java | 26 +- .../drill/exec/vector/complex/MapVector.java | 24 +- .../complex/RepeatedFixedWidthVectorLike.java | 16 - .../exec/vector/complex/RepeatedListVector.java | 45 +-- .../exec/vector/complex/RepeatedMapVector.java | 71 +++-- .../RepeatedVariableWidthVectorLike.java | 12 - .../apache/drill/exec/expr/ExpressionTest.java | 8 +- .../vector/complex/TestEmptyPopulation.java | 2 +- .../fn/TestJsonReaderWithSparseFiles.java | 2 +- .../drill/exec/proto/SchemaUserBitShared.java | 13 +- .../apache/drill/exec/proto/UserBitShared.java | 301 +++++++------------ .../drill/exec/proto/beans/SerializedField.java | 28 +- protocol/src/main/protobuf/UserBitShared.proto | 1 - 18 files changed, 209 insertions(+), 392 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java index 12dce25..6431d88 100644 --- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java +++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java @@ -228,23 +228,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector offsets.zeroVector(); mutator.reset(); } - - @Override - public int load(int dataBytes, int valueCount, int innerValueCount, DrillBuf buf){ - clear(); - int loaded = 0; - loaded += offsets.load(valueCount+1, buf.slice(loaded, buf.capacity() - loaded)); - loaded += values.load(dataBytes + 4*(innerValueCount + 1), innerValueCount, buf.slice(loaded, buf.capacity() - loaded)); - return loaded; - } - - @Override - public void load(SerializedField metadata, DrillBuf buffer) { - assert this.field.matches(metadata) : String.format("The field %s doesn't match the provided metadata %s.", this.field, metadata); - int loaded = load(metadata.getVarByteLength(), metadata.getGroupCount(), metadata.getValueCount(), buffer); - assert metadata.getBufferLength() == loaded : String.format("Expected to load %d bytes but actually loaded %d bytes", metadata.getBufferLength(), loaded); - } - + public int getByteCapacity(){ return values.getByteCapacity(); } @@ -255,7 +239,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector clear(); /* boolean to keep track if all the memory allocation were successful * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to + * buffers for multiple vectors. If one of the allocations failed we need to// * clear all the memory that we allocated */ boolean success = false; @@ -269,21 +253,6 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector offsets.zeroVector(); mutator.reset(); } - - public int load(int valueCount, int innerValueCount, DrillBuf buf){ - clear(); - int loaded = 0; - loaded += offsets.load(valueCount+1, buf.slice(loaded, buf.capacity() - loaded)); - loaded += values.load(innerValueCount, buf.slice(loaded, buf.capacity() - loaded)); - return loaded; - } - - @Override - public void load(SerializedField metadata, DrillBuf buffer) { - assert this.field.matches(metadata); - int loaded = load(metadata.getGroupCount(), metadata.getValueCount(), buffer); - assert metadata.getBufferLength() == loaded; - } </#if> http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java index bd41e10..50ae770 100644 --- a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java +++ b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java @@ -57,6 +57,9 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V private static final int INITIAL_BYTE_COUNT = 4096 * DEFAULT_RECORD_BYTE_COUNT; private static final int MIN_BYTE_COUNT = 4096; + public final static String OFFSETS_VECTOR_NAME = "offsets"; + private final static MaterializedField offsetsField = + MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(MinorType.UINT4)); private final UInt${type.width}Vector offsetVector; private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this); @@ -71,7 +74,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) { super(field, allocator); - this.offsetVector = new UInt${type.width}Vector(null, allocator); + this.offsetVector = new UInt${type.width}Vector(offsetsField, allocator); this.oAccessor = offsetVector.getAccessor(); this.accessor = new Accessor(); this.mutator = new Mutator(); http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java index de6f665..8e3b9e5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java @@ -93,7 +93,7 @@ public class RecordBatchLoader implements VectorAccessible, Iterable<VectorWrapp vector = TypeHelper.getNewVector(fieldDef, allocator); } - if (field.getValueCount() == 0 && (!field.hasGroupCount() || field.getGroupCount() == 0)) { + if (field.getValueCount() == 0) { AllocationHelper.allocate(vector, 0, 0, 0); } else { vector.load(field, buf.slice(bufOffset, field.getBufferLength())); http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java index ec409a3..8129668 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java @@ -35,7 +35,7 @@ public abstract class BaseValueVector implements ValueVector { public static final int INITIAL_VALUE_ALLOCATION = 4096; protected BaseValueVector(MaterializedField field, BufferAllocator allocator) { - this.field = field; + this.field = Preconditions.checkNotNull(field, "field cannot be null"); this.allocator = Preconditions.checkNotNull(allocator, "allocator cannot be null"); } http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java index af364bd..1df4b81 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java @@ -44,9 +44,11 @@ public abstract class AbstractMapVector extends AbstractContainerVector { super(field, allocator, callBack); // create the hierarchy of the child vectors based on the materialized field for (MaterializedField child : field.getChildren()) { - String fieldName = child.getLastName(); - ValueVector v = TypeHelper.getNewVector(child, allocator, callBack); - putVector(fieldName, v); + if (!child.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) { + String fieldName = child.getLastName(); + ValueVector v = TypeHelper.getNewVector(child, allocator, callBack); + putVector(fieldName, v); + } } } http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java index f292e4c..9bf6d85 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java @@ -40,10 +40,10 @@ import org.apache.drill.exec.vector.ZeroVector; public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector { public final static ValueVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE; - public final static String OFFSETS_VECTOR_NAME = "offsets"; - public final static String DATA_VECTOR_NAME = "data"; + public final static String OFFSETS_VECTOR_NAME = "$offsets$"; + public final static String DATA_VECTOR_NAME = "$data$"; - private final static MaterializedField offsetsField = + public final static MaterializedField OFFSETS_FIELD = MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(TypeProtos.MinorType.UINT4)); protected final UInt4Vector offsets; @@ -55,7 +55,7 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator, ValueVector vector) { super(field, allocator); - this.offsets = new UInt4Vector(offsetsField, allocator); + this.offsets = new UInt4Vector(OFFSETS_FIELD, allocator); this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null"); } @@ -110,8 +110,7 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements @Override protected UserBitShared.SerializedField.Builder getMetadataBuilder() { return super.getMetadataBuilder() - .setGroupCount(getAccessor().getValueCount()) - .setValueCount(getAccessor().getInnerValueCount()) + .addChild(offsets.getMetadata()) .addChild(vector.getMetadata()); } @@ -147,6 +146,21 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements return buffers; } + @Override + public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) { + final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0); + offsets.load(offsetMetadata, buffer); + + final UserBitShared.SerializedField vectorMetadata = metadata.getChild(1); + if (getDataVector() == DEFAULT_DATA_VECTOR) { + addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType())); + } + + final int offsetLength = offsetMetadata.getBufferLength(); + final int vectorLength = vectorMetadata.getBufferLength(); + vector.load(vectorMetadata, buffer.slice(offsetLength, vectorLength)); + } + /** * Returns 1 if inner vector is explicitly set via #addOrGetVector else 0 * http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java index d0f38c2..3032aac 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java @@ -245,28 +245,28 @@ public class MapVector extends AbstractMapVector { @Override public void load(SerializedField metadata, DrillBuf buf) { - List<SerializedField> fields = metadata.getChildList(); + final List<SerializedField> fields = metadata.getChildList(); valueCount = metadata.getValueCount(); int bufOffset = 0; - for (SerializedField fmd : fields) { - MaterializedField fieldDef = MaterializedField.create(fmd); + for (final SerializedField child : fields) { + final MaterializedField fieldDef = MaterializedField.create(child); - ValueVector v = getChild(fieldDef.getLastName()); - if (v == null) { + ValueVector vector = getChild(fieldDef.getLastName()); + if (vector == null) { // if we arrive here, we didn't have a matching vector. - v = TypeHelper.getNewVector(fieldDef, allocator); - putChild(fieldDef.getLastName(), v); + vector = TypeHelper.getNewVector(fieldDef, allocator); + putChild(fieldDef.getLastName(), vector); } - if (fmd.getValueCount() == 0 && (!fmd.hasGroupCount() || fmd.getGroupCount() == 0)) { - v.clear(); + if (child.getValueCount() == 0) { + vector.clear(); } else { - v.load(fmd, buf.slice(bufOffset, fmd.getBufferLength())); + vector.load(child, buf.slice(bufOffset, child.getBufferLength())); } - bufOffset += fmd.getBufferLength(); + bufOffset += child.getBufferLength(); } - Preconditions.checkArgument(bufOffset == buf.capacity()); + assert bufOffset == buf.capacity(); } @Override http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java index fb7ed2a..a6967f3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java @@ -17,8 +17,6 @@ */ package org.apache.drill.exec.vector.complex; -import io.netty.buffer.DrillBuf; - /** * A {@link org.apache.drill.exec.vector.ValueVector} mix-in that can be used in conjunction with * {@link RepeatedValueVector} subtypes. @@ -39,18 +37,4 @@ public interface RepeatedFixedWidthVectorLike { * @param buf Incoming buffer. * @return The number of bytes of the buffer that were consumed. */ - int load(int valueCount, int innerValueCount, DrillBuf buf); - -// public interface RepeatedAccessor extends Accessor { -// public int getGroupCount(); -// public int getValueCount(); -// public int getGroupSizeAtIndex(int index); -// public ValueVector getAllChildValues(); -// } -// -// public interface RepeatedMutator extends Mutator { -// public void setValueCounts(int parentValueCount, int childValueCount); -// public void setRepetitionAtIndexSafe(int index, int repetitionCount); -// public BaseDataValueVector getDataVector(); -// } } http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java index f6d3d88..f538399 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.vector.complex; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import io.netty.buffer.DrillBuf; import java.util.Collection; @@ -187,11 +188,6 @@ public class RepeatedListVector extends AbstractContainerVector } @Override - protected SerializedField.Builder getMetadataBuilder() { - return super.getMetadataBuilder(); - } - - @Override public TransferPair getTransferPair(FieldReference ref) { return makeTransferPair(new DelegateRepeatedVector(ref, allocator)); } @@ -216,23 +212,6 @@ public class RepeatedListVector extends AbstractContainerVector throw new UnsupportedOperationException(); } - @Override - public void load(SerializedField metadata, DrillBuf buffer) { - //TODO(DRILL-2997): get rid of the notion of "group count" completely - final int valueCount = metadata.getGroupCount(); - final int bufOffset = offsets.load(valueCount + 1, buffer); - final SerializedField childField = metadata.getChildList().get(0); - if (getDataVector() == DEFAULT_DATA_VECTOR) { - addOrGetVector(VectorDescriptor.create(childField.getMajorType())); - } - - if (childField.getValueCount() == 0) { - vector.clear(); - } else { - vector.load(childField, buffer.slice(bufOffset, childField.getBufferLength())); - } - } - public void copyFromSafe(int fromIndex, int thisIndex, DelegateRepeatedVector from) { if(ephPair == null || ephPair.target != from) { ephPair = DelegateTransferPair.class.cast(from.makeTransferPair(this)); @@ -282,13 +261,13 @@ public class RepeatedListVector extends AbstractContainerVector super(field, allocator, callBack); this.delegate = Preconditions.checkNotNull(delegate); - final Collection<MaterializedField> children = field.getChildren(); + final List<MaterializedField> children = Lists.newArrayList(field.getChildren()); final int childSize = children.size(); - // repeated list vector cannot have more than one child - assert childSize < 2; - final boolean hasChild = childSize == 1; + assert childSize < 3; + final boolean hasChild = childSize > 0; if (hasChild) { - final MaterializedField child = children.iterator().next(); + // the last field is data field + final MaterializedField child = children.get(childSize-1); addOrGetVector(VectorDescriptor.create(child)); } } @@ -431,11 +410,6 @@ public class RepeatedListVector extends AbstractContainerVector } @Override - public int load(int valueCount, int innerValueCount, DrillBuf buf) { - throw new UnsupportedOperationException(); - } - - @Override public VectorWithOrdinal getChildVectorWithOrdinal(String name) { if (name != null) { return null; @@ -443,15 +417,8 @@ public class RepeatedListVector extends AbstractContainerVector return new VectorWithOrdinal(delegate.getDataVector(), 0); } - public void copyFromSafe(int fromIndex, int thisIndex, RepeatedListVector from) { delegate.copyFromSafe(fromIndex, thisIndex, from.delegate); } - -// protected void setVector(ValueVector newVector) { -// vector = Preconditions.checkNotNull(newVector); -// getField().addChild(newVector.getField()); -// } - } http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java index 4617ede..97f5b39 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java @@ -68,7 +68,7 @@ public class RepeatedMapVector extends AbstractMapVector implements RepeatedValu public RepeatedMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){ super(field, allocator, callBack); - this.offsets = new UInt4Vector(null, allocator); + this.offsets = new UInt4Vector(BaseRepeatedValueVector.OFFSETS_FIELD, allocator); this.emptyPopulator = new EmptyValuePopulator(offsets); } @@ -433,43 +433,43 @@ public class RepeatedMapVector extends AbstractMapVector implements RepeatedValu @Override - public void load(SerializedField metadata, DrillBuf buf) { - List<SerializedField> fields = metadata.getChildList(); - - int bufOffset = offsets.load(metadata.getGroupCount()+1, buf); - - for (SerializedField fmd : fields) { - MaterializedField fieldDef = MaterializedField.create(fmd); - ValueVector v = getChild(fieldDef.getLastName()); - if (v == null) { + public void load(SerializedField metadata, DrillBuf buffer) { + final List<SerializedField> children = metadata.getChildList(); + + final SerializedField offsetField = children.get(0); + offsets.load(offsetField, buffer); + int bufOffset = offsetField.getBufferLength(); + + for (int i=1; i<children.size(); i++) { + final SerializedField child = children.get(i); + final MaterializedField fieldDef = MaterializedField.create(child); + ValueVector vector = getChild(fieldDef.getLastName()); + if (vector == null) { // if we arrive here, we didn't have a matching vector. - v = TypeHelper.getNewVector(fieldDef, allocator); - putChild(fieldDef.getLastName(), v); - } - if (fmd.getValueCount() == 0 && (!fmd.hasGroupCount() || fmd.getGroupCount() == 0)) { - v.clear(); - } else { - v.load(fmd, buf.slice(bufOffset, fmd.getBufferLength())); + vector = TypeHelper.getNewVector(fieldDef, allocator); + putChild(fieldDef.getLastName(), vector); } - bufOffset += fmd.getBufferLength(); + final int vectorLength = child.getBufferLength(); + vector.load(child, buffer.slice(bufOffset, vectorLength)); + bufOffset += vectorLength; } - Preconditions.checkArgument(bufOffset == buf.capacity()); + assert bufOffset == buffer.capacity(); } @Override public SerializedField getMetadata() { - SerializedField.Builder b = getField() // + SerializedField.Builder builder = getField() // .getAsBuilder() // .setBufferLength(getBufferSize()) // - .setGroupCount(accessor.getValueCount()) // while we don't need to actually read this on load, we need it to make sure we don't skip deserialization of this vector - .setValueCount(accessor.getInnerValueCount()); - for (ValueVector v : getChildren()) { - b.addChild(v.getMetadata()); + .setValueCount(accessor.getValueCount()); + builder.addChild(offsets.getMetadata()); + for (final ValueVector child : getChildren()) { + builder.addChild(child.getMetadata()); } - return b.build(); + return builder.build(); } @Override @@ -481,21 +481,23 @@ public class RepeatedMapVector extends AbstractMapVector implements RepeatedValu @Override public Object getObject(int index) { - List<Object> l = new JsonStringArrayList(); + final List<Object> list = new JsonStringArrayList(); int end = offsets.getAccessor().get(index+1); String fieldName; for (int i = offsets.getAccessor().get(index); i < end; i++) { Map<String, Object> vv = Maps.newLinkedHashMap(); for (MaterializedField field:getField().getChildren()) { - fieldName = field.getLastName(); - Object value = getChild(fieldName).getAccessor().getObject(i); - if (value != null) { - vv.put(fieldName, value); + if (!field.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) { + fieldName = field.getLastName(); + final Object value = getChild(fieldName).getAccessor().getObject(i); + if (value != null) { + vv.put(fieldName, value); + } } } - l.add(vv); + list.add(vv); } - return l; + return list; } @Override @@ -595,9 +597,4 @@ public class RepeatedMapVector extends AbstractMapVector implements RepeatedValu vector.clear(); } } - - @Override - public int load(int valueCount, int innerValueCount, DrillBuf buf) { - throw new UnsupportedOperationException(); - } } http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java index c57143e..67954e7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java @@ -17,8 +17,6 @@ */ package org.apache.drill.exec.vector.complex; -import io.netty.buffer.DrillBuf; - public interface RepeatedVariableWidthVectorLike { /** * Allocate a new memory space for this vector. Must be called prior to using the ValueVector. @@ -34,14 +32,4 @@ public interface RepeatedVariableWidthVectorLike { * @return */ int getByteCapacity(); - - /** - * Load the records in the provided buffer based on the given number of values. - * @param dataBytes The number of bytes associated with the data array. - * @param parentValueCount Number of separate repeating groupings. - * @param childValueCount Number of supported values in the vector. - * @param buf Incoming buffer. - * @return The number of bytes of the buffer that were consumed. - */ - int load(int dataBytes, int parentValueCount, int childValueCount, DrillBuf buf); } http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java index 239a099..5e50189 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java @@ -36,6 +36,8 @@ import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.expression.parser.ExprLexer; import org.apache.drill.common.expression.parser.ExprParser; import org.apache.drill.common.expression.parser.ExprParser.parse_return; +import org.apache.drill.common.types.MajorType; +import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; import org.apache.drill.exec.ExecTest; @@ -43,6 +45,7 @@ import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.memory.TopLevelAllocator; import org.apache.drill.exec.physical.impl.project.Projector; +import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.RecordBatch; import org.apache.drill.exec.record.TypedFieldId; import org.apache.drill.exec.record.VectorWrapper; @@ -73,7 +76,8 @@ public class ExpressionTest extends ExecTest { @Test public void testSpecial(final @Injectable RecordBatch batch, @Injectable ValueVector vector) throws Exception { - final TypedFieldId tfid = new TypedFieldId(Types.optional(MinorType.INT), false, 0); + final TypeProtos.MajorType type = Types.optional(MinorType.INT); + final TypedFieldId tfid = new TypedFieldId(type, false, 0); new NonStrictExpectations() { @NonStrict VectorWrapper<?> wrapper; @@ -83,7 +87,7 @@ public class ExpressionTest extends ExecTest { batch.getValueAccessorById(IntVector.class, tfid.getFieldIds()); result = wrapper; wrapper.getValueVector(); - result = new IntVector(null, new TopLevelAllocator(0)); + result = new IntVector(MaterializedField.create("result", type), new TopLevelAllocator(0)); } }; http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/TestEmptyPopulation.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/TestEmptyPopulation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/TestEmptyPopulation.java index 06a73e2..f7f2706 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/TestEmptyPopulation.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/TestEmptyPopulation.java @@ -41,7 +41,7 @@ public class TestEmptyPopulation extends BaseTestQuery { @Before public void initialize() { - offsets = new UInt4Vector(null, allocator); + offsets = new UInt4Vector(BaseRepeatedValueVector.OFFSETS_FIELD, allocator); offsets.allocateNewSafe(); accessor = offsets.getAccessor(); mutator = offsets.getMutator(); http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java index 544b962..808d98c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java @@ -30,7 +30,7 @@ import org.apache.drill.exec.vector.ValueVector; import org.junit.Test; public class TestJsonReaderWithSparseFiles extends BaseTestQuery { - private static interface Function<T> { + private interface Function<T> { void apply(T param); } http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java ---------------------------------------------------------------------- diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java index bee2a3d..8c0b66a 100644 --- a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java +++ b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java @@ -1090,10 +1090,8 @@ public final class SchemaUserBitShared output.writeInt32(4, message.getValueCount(), false); if(message.hasVarByteLength()) output.writeInt32(5, message.getVarByteLength(), false); - if(message.hasGroupCount()) - output.writeInt32(6, message.getGroupCount(), false); if(message.hasBufferLength()) - output.writeInt32(7, message.getBufferLength(), false); + output.writeInt32(6, message.getBufferLength(), false); } public boolean isInitialized(org.apache.drill.exec.proto.UserBitShared.SerializedField message) { @@ -1152,9 +1150,6 @@ public final class SchemaUserBitShared builder.setVarByteLength(input.readInt32()); break; case 6: - builder.setGroupCount(input.readInt32()); - break; - case 7: builder.setBufferLength(input.readInt32()); break; default: @@ -1202,8 +1197,7 @@ public final class SchemaUserBitShared case 3: return "child"; case 4: return "valueCount"; case 5: return "varByteLength"; - case 6: return "groupCount"; - case 7: return "bufferLength"; + case 6: return "bufferLength"; default: return null; } } @@ -1220,8 +1214,7 @@ public final class SchemaUserBitShared fieldMap.put("child", 3); fieldMap.put("valueCount", 4); fieldMap.put("varByteLength", 5); - fieldMap.put("groupCount", 6); - fieldMap.put("bufferLength", 7); + fieldMap.put("bufferLength", 6); } } http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java ---------------------------------------------------------------------- diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java index 92afa4f..b8acf05 100644 --- a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java +++ b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java @@ -8136,32 +8136,22 @@ public final class UserBitShared { */ int getVarByteLength(); - // optional int32 group_count = 6; + // optional int32 buffer_length = 6; /** - * <code>optional int32 group_count = 6;</code> + * <code>optional int32 buffer_length = 6;</code> * * <pre> - * number of groups. (number of repeated records) + *optional int32 group_count = 6; // number of groups. (number of repeated records) * </pre> */ - boolean hasGroupCount(); + boolean hasBufferLength(); /** - * <code>optional int32 group_count = 6;</code> + * <code>optional int32 buffer_length = 6;</code> * * <pre> - * number of groups. (number of repeated records) + *optional int32 group_count = 6; // number of groups. (number of repeated records) * </pre> */ - int getGroupCount(); - - // optional int32 buffer_length = 7; - /** - * <code>optional int32 buffer_length = 7;</code> - */ - boolean hasBufferLength(); - /** - * <code>optional int32 buffer_length = 7;</code> - */ int getBufferLength(); } /** @@ -8261,11 +8251,6 @@ public final class UserBitShared { } case 48: { bitField0_ |= 0x00000010; - groupCount_ = input.readInt32(); - break; - } - case 56: { - bitField0_ |= 0x00000020; bufferLength_ = input.readInt32(); break; } @@ -8456,42 +8441,26 @@ public final class UserBitShared { return varByteLength_; } - // optional int32 group_count = 6; - public static final int GROUP_COUNT_FIELD_NUMBER = 6; - private int groupCount_; + // optional int32 buffer_length = 6; + public static final int BUFFER_LENGTH_FIELD_NUMBER = 6; + private int bufferLength_; /** - * <code>optional int32 group_count = 6;</code> + * <code>optional int32 buffer_length = 6;</code> * * <pre> - * number of groups. (number of repeated records) + *optional int32 group_count = 6; // number of groups. (number of repeated records) * </pre> */ - public boolean hasGroupCount() { + public boolean hasBufferLength() { return ((bitField0_ & 0x00000010) == 0x00000010); } /** - * <code>optional int32 group_count = 6;</code> + * <code>optional int32 buffer_length = 6;</code> * * <pre> - * number of groups. (number of repeated records) + *optional int32 group_count = 6; // number of groups. (number of repeated records) * </pre> */ - public int getGroupCount() { - return groupCount_; - } - - // optional int32 buffer_length = 7; - public static final int BUFFER_LENGTH_FIELD_NUMBER = 7; - private int bufferLength_; - /** - * <code>optional int32 buffer_length = 7;</code> - */ - public boolean hasBufferLength() { - return ((bitField0_ & 0x00000020) == 0x00000020); - } - /** - * <code>optional int32 buffer_length = 7;</code> - */ public int getBufferLength() { return bufferLength_; } @@ -8502,7 +8471,6 @@ public final class UserBitShared { child_ = java.util.Collections.emptyList(); valueCount_ = 0; varByteLength_ = 0; - groupCount_ = 0; bufferLength_ = 0; } private byte memoizedIsInitialized = -1; @@ -8533,10 +8501,7 @@ public final class UserBitShared { output.writeInt32(5, varByteLength_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { - output.writeInt32(6, groupCount_); - } - if (((bitField0_ & 0x00000020) == 0x00000020)) { - output.writeInt32(7, bufferLength_); + output.writeInt32(6, bufferLength_); } getUnknownFields().writeTo(output); } @@ -8569,11 +8534,7 @@ public final class UserBitShared { } if (((bitField0_ & 0x00000010) == 0x00000010)) { size += com.google.protobuf.CodedOutputStream - .computeInt32Size(6, groupCount_); - } - if (((bitField0_ & 0x00000020) == 0x00000020)) { - size += com.google.protobuf.CodedOutputStream - .computeInt32Size(7, bufferLength_); + .computeInt32Size(6, bufferLength_); } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; @@ -8716,10 +8677,8 @@ public final class UserBitShared { bitField0_ = (bitField0_ & ~0x00000008); varByteLength_ = 0; bitField0_ = (bitField0_ & ~0x00000010); - groupCount_ = 0; - bitField0_ = (bitField0_ & ~0x00000020); bufferLength_ = 0; - bitField0_ = (bitField0_ & ~0x00000040); + bitField0_ = (bitField0_ & ~0x00000020); return this; } @@ -8784,10 +8743,6 @@ public final class UserBitShared { if (((from_bitField0_ & 0x00000020) == 0x00000020)) { to_bitField0_ |= 0x00000010; } - result.groupCount_ = groupCount_; - if (((from_bitField0_ & 0x00000040) == 0x00000040)) { - to_bitField0_ |= 0x00000020; - } result.bufferLength_ = bufferLength_; result.bitField0_ = to_bitField0_; onBuilt(); @@ -8843,9 +8798,6 @@ public final class UserBitShared { if (other.hasVarByteLength()) { setVarByteLength(other.getVarByteLength()); } - if (other.hasGroupCount()) { - setGroupCount(other.getGroupCount()); - } if (other.hasBufferLength()) { setBufferLength(other.getBufferLength()); } @@ -9524,83 +9476,50 @@ public final class UserBitShared { return this; } - // optional int32 group_count = 6; - private int groupCount_ ; + // optional int32 buffer_length = 6; + private int bufferLength_ ; /** - * <code>optional int32 group_count = 6;</code> + * <code>optional int32 buffer_length = 6;</code> * * <pre> - * number of groups. (number of repeated records) + *optional int32 group_count = 6; // number of groups. (number of repeated records) * </pre> */ - public boolean hasGroupCount() { + public boolean hasBufferLength() { return ((bitField0_ & 0x00000020) == 0x00000020); } /** - * <code>optional int32 group_count = 6;</code> + * <code>optional int32 buffer_length = 6;</code> * * <pre> - * number of groups. (number of repeated records) + *optional int32 group_count = 6; // number of groups. (number of repeated records) * </pre> */ - public int getGroupCount() { - return groupCount_; + public int getBufferLength() { + return bufferLength_; } /** - * <code>optional int32 group_count = 6;</code> + * <code>optional int32 buffer_length = 6;</code> * * <pre> - * number of groups. (number of repeated records) + *optional int32 group_count = 6; // number of groups. (number of repeated records) * </pre> */ - public Builder setGroupCount(int value) { + public Builder setBufferLength(int value) { bitField0_ |= 0x00000020; - groupCount_ = value; + bufferLength_ = value; onChanged(); return this; } /** - * <code>optional int32 group_count = 6;</code> + * <code>optional int32 buffer_length = 6;</code> * * <pre> - * number of groups. (number of repeated records) + *optional int32 group_count = 6; // number of groups. (number of repeated records) * </pre> */ - public Builder clearGroupCount() { - bitField0_ = (bitField0_ & ~0x00000020); - groupCount_ = 0; - onChanged(); - return this; - } - - // optional int32 buffer_length = 7; - private int bufferLength_ ; - /** - * <code>optional int32 buffer_length = 7;</code> - */ - public boolean hasBufferLength() { - return ((bitField0_ & 0x00000040) == 0x00000040); - } - /** - * <code>optional int32 buffer_length = 7;</code> - */ - public int getBufferLength() { - return bufferLength_; - } - /** - * <code>optional int32 buffer_length = 7;</code> - */ - public Builder setBufferLength(int value) { - bitField0_ |= 0x00000040; - bufferLength_ = value; - onChanged(); - return this; - } - /** - * <code>optional int32 buffer_length = 7;</code> - */ public Builder clearBufferLength() { - bitField0_ = (bitField0_ & ~0x00000040); + bitField0_ = (bitField0_ & ~0x00000020); bufferLength_ = 0; onChanged(); return this; @@ -20894,86 +20813,86 @@ public final class UserBitShared { "(\010\"\205\001\n\010NamePart\022(\n\004type\030\001 \001(\0162\032.exec.sha" + "red.NamePart.Type\022\014\n\004name\030\002 \001(\t\022$\n\005child", "\030\003 \001(\0132\025.exec.shared.NamePart\"\033\n\004Type\022\010\n" + - "\004NAME\020\000\022\t\n\005ARRAY\020\001\"\351\001\n\017SerializedField\022%" + + "\004NAME\020\000\022\t\n\005ARRAY\020\001\"\324\001\n\017SerializedField\022%" + "\n\nmajor_type\030\001 \001(\0132\021.common.MajorType\022(\n" + "\tname_part\030\002 \001(\0132\025.exec.shared.NamePart\022" + "+\n\005child\030\003 \003(\0132\034.exec.shared.SerializedF" + "ield\022\023\n\013value_count\030\004 \001(\005\022\027\n\017var_byte_le" + - "ngth\030\005 \001(\005\022\023\n\013group_count\030\006 \001(\005\022\025\n\rbuffe" + - "r_length\030\007 \001(\005\"7\n\nNodeStatus\022\017\n\007node_id\030" + - "\001 \001(\005\022\030\n\020memory_footprint\030\002 \001(\003\"\206\002\n\013Quer" + - "yResult\0228\n\013query_state\030\001 \001(\0162#.exec.shar", - "ed.QueryResult.QueryState\022&\n\010query_id\030\002 " + - "\001(\0132\024.exec.shared.QueryId\022(\n\005error\030\003 \003(\013" + - "2\031.exec.shared.DrillPBError\"k\n\nQueryStat" + - "e\022\013\n\007PENDING\020\000\022\013\n\007RUNNING\020\001\022\r\n\tCOMPLETED" + - "\020\002\022\014\n\010CANCELED\020\003\022\n\n\006FAILED\020\004\022\032\n\026CANCELLA" + - "TION_REQUESTED\020\005\"p\n\tQueryData\022&\n\010query_i" + - "d\030\001 \001(\0132\024.exec.shared.QueryId\022\021\n\trow_cou" + - "nt\030\002 \001(\005\022(\n\003def\030\003 \001(\0132\033.exec.shared.Reco" + - "rdBatchDef\"\227\001\n\tQueryInfo\022\r\n\005query\030\001 \001(\t\022" + - "\r\n\005start\030\002 \001(\003\0222\n\005state\030\003 \001(\0162#.exec.sha", - "red.QueryResult.QueryState\022\017\n\004user\030\004 \001(\t" + - ":\001-\022\'\n\007foreman\030\005 \001(\0132\026.exec.DrillbitEndp" + - "oint\"\272\003\n\014QueryProfile\022 \n\002id\030\001 \001(\0132\024.exec" + - ".shared.QueryId\022$\n\004type\030\002 \001(\0162\026.exec.sha" + - "red.QueryType\022\r\n\005start\030\003 \001(\003\022\013\n\003end\030\004 \001(" + - "\003\022\r\n\005query\030\005 \001(\t\022\014\n\004plan\030\006 \001(\t\022\'\n\007forema" + - "n\030\007 \001(\0132\026.exec.DrillbitEndpoint\0222\n\005state" + - "\030\010 \001(\0162#.exec.shared.QueryResult.QuerySt" + - "ate\022\027\n\017total_fragments\030\t \001(\005\022\032\n\022finished" + - "_fragments\030\n \001(\005\022;\n\020fragment_profile\030\013 \003", - "(\0132!.exec.shared.MajorFragmentProfile\022\017\n" + - "\004user\030\014 \001(\t:\001-\022\r\n\005error\030\r \001(\t\022\024\n\014verbose" + - "Error\030\016 \001(\t\022\020\n\010error_id\030\017 \001(\t\022\022\n\nerror_n" + - "ode\030\020 \001(\t\"t\n\024MajorFragmentProfile\022\031\n\021maj" + - "or_fragment_id\030\001 \001(\005\022A\n\026minor_fragment_p" + - "rofile\030\002 \003(\0132!.exec.shared.MinorFragment" + - "Profile\"\350\002\n\024MinorFragmentProfile\022)\n\005stat" + - "e\030\001 \001(\0162\032.exec.shared.FragmentState\022(\n\005e" + - "rror\030\002 \001(\0132\031.exec.shared.DrillPBError\022\031\n" + - "\021minor_fragment_id\030\003 \001(\005\0226\n\020operator_pro", - "file\030\004 \003(\0132\034.exec.shared.OperatorProfile" + - "\022\022\n\nstart_time\030\005 \001(\003\022\020\n\010end_time\030\006 \001(\003\022\023" + - "\n\013memory_used\030\007 \001(\003\022\027\n\017max_memory_used\030\010" + - " \001(\003\022(\n\010endpoint\030\t \001(\0132\026.exec.DrillbitEn" + - "dpoint\022\023\n\013last_update\030\n \001(\003\022\025\n\rlast_prog" + - "ress\030\013 \001(\003\"\377\001\n\017OperatorProfile\0221\n\rinput_" + - "profile\030\001 \003(\0132\032.exec.shared.StreamProfil" + - "e\022\023\n\013operator_id\030\003 \001(\005\022\025\n\roperator_type\030" + - "\004 \001(\005\022\023\n\013setup_nanos\030\005 \001(\003\022\025\n\rprocess_na" + - "nos\030\006 \001(\003\022#\n\033peak_local_memory_allocated", - "\030\007 \001(\003\022(\n\006metric\030\010 \003(\0132\030.exec.shared.Met" + - "ricValue\022\022\n\nwait_nanos\030\t \001(\003\"B\n\rStreamPr" + - "ofile\022\017\n\007records\030\001 \001(\003\022\017\n\007batches\030\002 \001(\003\022" + - "\017\n\007schemas\030\003 \001(\003\"J\n\013MetricValue\022\021\n\tmetri" + - "c_id\030\001 \001(\005\022\022\n\nlong_value\030\002 \001(\003\022\024\n\014double" + - "_value\030\003 \001(\001*5\n\nRpcChannel\022\017\n\013BIT_CONTRO" + - "L\020\000\022\014\n\010BIT_DATA\020\001\022\010\n\004USER\020\002*/\n\tQueryType" + - "\022\007\n\003SQL\020\001\022\013\n\007LOGICAL\020\002\022\014\n\010PHYSICAL\020\003*\207\001\n" + - "\rFragmentState\022\013\n\007SENDING\020\000\022\027\n\023AWAITING_" + - "ALLOCATION\020\001\022\013\n\007RUNNING\020\002\022\014\n\010FINISHED\020\003\022", - "\r\n\tCANCELLED\020\004\022\n\n\006FAILED\020\005\022\032\n\026CANCELLATI" + - "ON_REQUESTED\020\006*\335\005\n\020CoreOperatorType\022\021\n\rS" + - "INGLE_SENDER\020\000\022\024\n\020BROADCAST_SENDER\020\001\022\n\n\006" + - "FILTER\020\002\022\022\n\016HASH_AGGREGATE\020\003\022\r\n\tHASH_JOI" + - "N\020\004\022\016\n\nMERGE_JOIN\020\005\022\031\n\025HASH_PARTITION_SE" + - "NDER\020\006\022\t\n\005LIMIT\020\007\022\024\n\020MERGING_RECEIVER\020\010\022" + - "\034\n\030ORDERED_PARTITION_SENDER\020\t\022\013\n\007PROJECT" + - "\020\n\022\026\n\022UNORDERED_RECEIVER\020\013\022\020\n\014RANGE_SEND" + - "ER\020\014\022\n\n\006SCREEN\020\r\022\034\n\030SELECTION_VECTOR_REM" + - "OVER\020\016\022\027\n\023STREAMING_AGGREGATE\020\017\022\016\n\nTOP_N", - "_SORT\020\020\022\021\n\rEXTERNAL_SORT\020\021\022\t\n\005TRACE\020\022\022\t\n" + - "\005UNION\020\023\022\014\n\010OLD_SORT\020\024\022\032\n\026PARQUET_ROW_GR" + - "OUP_SCAN\020\025\022\021\n\rHIVE_SUB_SCAN\020\026\022\025\n\021SYSTEM_" + - "TABLE_SCAN\020\027\022\021\n\rMOCK_SUB_SCAN\020\030\022\022\n\016PARQU" + - "ET_WRITER\020\031\022\023\n\017DIRECT_SUB_SCAN\020\032\022\017\n\013TEXT" + - "_WRITER\020\033\022\021\n\rTEXT_SUB_SCAN\020\034\022\021\n\rJSON_SUB" + - "_SCAN\020\035\022\030\n\024INFO_SCHEMA_SUB_SCAN\020\036\022\023\n\017COM" + - "PLEX_TO_JSON\020\037\022\025\n\021PRODUCER_CONSUMER\020 \022\022\n" + - "\016HBASE_SUB_SCAN\020!\022\n\n\006WINDOW\020\"\022\024\n\020NESTED_" + - "LOOP_JOIN\020#\022\021\n\rAVRO_SUB_SCAN\020$B.\n\033org.ap", - "ache.drill.exec.protoB\rUserBitSharedH\001" + "ngth\030\005 \001(\005\022\025\n\rbuffer_length\030\006 \001(\005\"7\n\nNod" + + "eStatus\022\017\n\007node_id\030\001 \001(\005\022\030\n\020memory_footp" + + "rint\030\002 \001(\003\"\206\002\n\013QueryResult\0228\n\013query_stat" + + "e\030\001 \001(\0162#.exec.shared.QueryResult.QueryS", + "tate\022&\n\010query_id\030\002 \001(\0132\024.exec.shared.Que" + + "ryId\022(\n\005error\030\003 \003(\0132\031.exec.shared.DrillP" + + "BError\"k\n\nQueryState\022\013\n\007PENDING\020\000\022\013\n\007RUN" + + "NING\020\001\022\r\n\tCOMPLETED\020\002\022\014\n\010CANCELED\020\003\022\n\n\006F" + + "AILED\020\004\022\032\n\026CANCELLATION_REQUESTED\020\005\"p\n\tQ" + + "ueryData\022&\n\010query_id\030\001 \001(\0132\024.exec.shared" + + ".QueryId\022\021\n\trow_count\030\002 \001(\005\022(\n\003def\030\003 \001(\013" + + "2\033.exec.shared.RecordBatchDef\"\227\001\n\tQueryI" + + "nfo\022\r\n\005query\030\001 \001(\t\022\r\n\005start\030\002 \001(\003\0222\n\005sta" + + "te\030\003 \001(\0162#.exec.shared.QueryResult.Query", + "State\022\017\n\004user\030\004 \001(\t:\001-\022\'\n\007foreman\030\005 \001(\0132" + + "\026.exec.DrillbitEndpoint\"\272\003\n\014QueryProfile" + + "\022 \n\002id\030\001 \001(\0132\024.exec.shared.QueryId\022$\n\004ty" + + "pe\030\002 \001(\0162\026.exec.shared.QueryType\022\r\n\005star" + + "t\030\003 \001(\003\022\013\n\003end\030\004 \001(\003\022\r\n\005query\030\005 \001(\t\022\014\n\004p" + + "lan\030\006 \001(\t\022\'\n\007foreman\030\007 \001(\0132\026.exec.Drillb" + + "itEndpoint\0222\n\005state\030\010 \001(\0162#.exec.shared." + + "QueryResult.QueryState\022\027\n\017total_fragment" + + "s\030\t \001(\005\022\032\n\022finished_fragments\030\n \001(\005\022;\n\020f" + + "ragment_profile\030\013 \003(\0132!.exec.shared.Majo", + "rFragmentProfile\022\017\n\004user\030\014 \001(\t:\001-\022\r\n\005err" + + "or\030\r \001(\t\022\024\n\014verboseError\030\016 \001(\t\022\020\n\010error_" + + "id\030\017 \001(\t\022\022\n\nerror_node\030\020 \001(\t\"t\n\024MajorFra" + + "gmentProfile\022\031\n\021major_fragment_id\030\001 \001(\005\022" + + "A\n\026minor_fragment_profile\030\002 \003(\0132!.exec.s" + + "hared.MinorFragmentProfile\"\350\002\n\024MinorFrag" + + "mentProfile\022)\n\005state\030\001 \001(\0162\032.exec.shared" + + ".FragmentState\022(\n\005error\030\002 \001(\0132\031.exec.sha" + + "red.DrillPBError\022\031\n\021minor_fragment_id\030\003 " + + "\001(\005\0226\n\020operator_profile\030\004 \003(\0132\034.exec.sha", + "red.OperatorProfile\022\022\n\nstart_time\030\005 \001(\003\022" + + "\020\n\010end_time\030\006 \001(\003\022\023\n\013memory_used\030\007 \001(\003\022\027" + + "\n\017max_memory_used\030\010 \001(\003\022(\n\010endpoint\030\t \001(" + + "\0132\026.exec.DrillbitEndpoint\022\023\n\013last_update" + + "\030\n \001(\003\022\025\n\rlast_progress\030\013 \001(\003\"\377\001\n\017Operat" + + "orProfile\0221\n\rinput_profile\030\001 \003(\0132\032.exec." + + "shared.StreamProfile\022\023\n\013operator_id\030\003 \001(" + + "\005\022\025\n\roperator_type\030\004 \001(\005\022\023\n\013setup_nanos\030" + + "\005 \001(\003\022\025\n\rprocess_nanos\030\006 \001(\003\022#\n\033peak_loc" + + "al_memory_allocated\030\007 \001(\003\022(\n\006metric\030\010 \003(", + "\0132\030.exec.shared.MetricValue\022\022\n\nwait_nano" + + "s\030\t \001(\003\"B\n\rStreamProfile\022\017\n\007records\030\001 \001(" + + "\003\022\017\n\007batches\030\002 \001(\003\022\017\n\007schemas\030\003 \001(\003\"J\n\013M" + + "etricValue\022\021\n\tmetric_id\030\001 \001(\005\022\022\n\nlong_va" + + "lue\030\002 \001(\003\022\024\n\014double_value\030\003 \001(\001*5\n\nRpcCh" + + "annel\022\017\n\013BIT_CONTROL\020\000\022\014\n\010BIT_DATA\020\001\022\010\n\004" + + "USER\020\002*/\n\tQueryType\022\007\n\003SQL\020\001\022\013\n\007LOGICAL\020" + + "\002\022\014\n\010PHYSICAL\020\003*\207\001\n\rFragmentState\022\013\n\007SEN" + + "DING\020\000\022\027\n\023AWAITING_ALLOCATION\020\001\022\013\n\007RUNNI" + + "NG\020\002\022\014\n\010FINISHED\020\003\022\r\n\tCANCELLED\020\004\022\n\n\006FAI", + "LED\020\005\022\032\n\026CANCELLATION_REQUESTED\020\006*\335\005\n\020Co" + + "reOperatorType\022\021\n\rSINGLE_SENDER\020\000\022\024\n\020BRO" + + "ADCAST_SENDER\020\001\022\n\n\006FILTER\020\002\022\022\n\016HASH_AGGR" + + "EGATE\020\003\022\r\n\tHASH_JOIN\020\004\022\016\n\nMERGE_JOIN\020\005\022\031" + + "\n\025HASH_PARTITION_SENDER\020\006\022\t\n\005LIMIT\020\007\022\024\n\020" + + "MERGING_RECEIVER\020\010\022\034\n\030ORDERED_PARTITION_" + + "SENDER\020\t\022\013\n\007PROJECT\020\n\022\026\n\022UNORDERED_RECEI" + + "VER\020\013\022\020\n\014RANGE_SENDER\020\014\022\n\n\006SCREEN\020\r\022\034\n\030S" + + "ELECTION_VECTOR_REMOVER\020\016\022\027\n\023STREAMING_A" + + "GGREGATE\020\017\022\016\n\nTOP_N_SORT\020\020\022\021\n\rEXTERNAL_S", + "ORT\020\021\022\t\n\005TRACE\020\022\022\t\n\005UNION\020\023\022\014\n\010OLD_SORT\020" + + "\024\022\032\n\026PARQUET_ROW_GROUP_SCAN\020\025\022\021\n\rHIVE_SU" + + "B_SCAN\020\026\022\025\n\021SYSTEM_TABLE_SCAN\020\027\022\021\n\rMOCK_" + + "SUB_SCAN\020\030\022\022\n\016PARQUET_WRITER\020\031\022\023\n\017DIRECT" + + "_SUB_SCAN\020\032\022\017\n\013TEXT_WRITER\020\033\022\021\n\rTEXT_SUB" + + "_SCAN\020\034\022\021\n\rJSON_SUB_SCAN\020\035\022\030\n\024INFO_SCHEM" + + "A_SUB_SCAN\020\036\022\023\n\017COMPLEX_TO_JSON\020\037\022\025\n\021PRO" + + "DUCER_CONSUMER\020 \022\022\n\016HBASE_SUB_SCAN\020!\022\n\n\006" + + "WINDOW\020\"\022\024\n\020NESTED_LOOP_JOIN\020#\022\021\n\rAVRO_S" + + "UB_SCAN\020$B.\n\033org.apache.drill.exec.proto", + "B\rUserBitSharedH\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -21033,7 +20952,7 @@ public final class UserBitShared { internal_static_exec_shared_SerializedField_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_exec_shared_SerializedField_descriptor, - new java.lang.String[] { "MajorType", "NamePart", "Child", "ValueCount", "VarByteLength", "GroupCount", "BufferLength", }); + new java.lang.String[] { "MajorType", "NamePart", "Child", "ValueCount", "VarByteLength", "BufferLength", }); internal_static_exec_shared_NodeStatus_descriptor = getDescriptor().getMessageTypes().get(9); internal_static_exec_shared_NodeStatus_fieldAccessorTable = new http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/protocol/src/main/java/org/apache/drill/exec/proto/beans/SerializedField.java ---------------------------------------------------------------------- diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/beans/SerializedField.java b/protocol/src/main/java/org/apache/drill/exec/proto/beans/SerializedField.java index 699097a..6d01a69 100644 --- a/protocol/src/main/java/org/apache/drill/exec/proto/beans/SerializedField.java +++ b/protocol/src/main/java/org/apache/drill/exec/proto/beans/SerializedField.java @@ -54,7 +54,6 @@ public final class SerializedField implements Externalizable, Message<Serialized private List<SerializedField> child; private int valueCount; private int varByteLength; - private int groupCount; private int bufferLength; public SerializedField() @@ -129,19 +128,6 @@ public final class SerializedField implements Externalizable, Message<Serialized return this; } - // groupCount - - public int getGroupCount() - { - return groupCount; - } - - public SerializedField setGroupCount(int groupCount) - { - this.groupCount = groupCount; - return this; - } - // bufferLength public int getBufferLength() @@ -230,9 +216,6 @@ public final class SerializedField implements Externalizable, Message<Serialized message.varByteLength = input.readInt32(); break; case 6: - message.groupCount = input.readInt32(); - break; - case 7: message.bufferLength = input.readInt32(); break; default: @@ -268,11 +251,8 @@ public final class SerializedField implements Externalizable, Message<Serialized if(message.varByteLength != 0) output.writeInt32(5, message.varByteLength, false); - if(message.groupCount != 0) - output.writeInt32(6, message.groupCount, false); - if(message.bufferLength != 0) - output.writeInt32(7, message.bufferLength, false); + output.writeInt32(6, message.bufferLength, false); } public String getFieldName(int number) @@ -284,8 +264,7 @@ public final class SerializedField implements Externalizable, Message<Serialized case 3: return "child"; case 4: return "valueCount"; case 5: return "varByteLength"; - case 6: return "groupCount"; - case 7: return "bufferLength"; + case 6: return "bufferLength"; default: return null; } } @@ -304,8 +283,7 @@ public final class SerializedField implements Externalizable, Message<Serialized __fieldMap.put("child", 3); __fieldMap.put("valueCount", 4); __fieldMap.put("varByteLength", 5); - __fieldMap.put("groupCount", 6); - __fieldMap.put("bufferLength", 7); + __fieldMap.put("bufferLength", 6); } } http://git-wip-us.apache.org/repos/asf/drill/blob/68c933c7/protocol/src/main/protobuf/UserBitShared.proto ---------------------------------------------------------------------- diff --git a/protocol/src/main/protobuf/UserBitShared.proto b/protocol/src/main/protobuf/UserBitShared.proto index 68c8612..0451fd2 100644 --- a/protocol/src/main/protobuf/UserBitShared.proto +++ b/protocol/src/main/protobuf/UserBitShared.proto @@ -139,7 +139,6 @@ message SerializedField { optional int32 value_count = 4; optional int32 var_byte_length = 5; - optional int32 group_count = 6; // number of groups. (number of repeated records) optional int32 buffer_length = 7; }
