Repository: drill Updated Branches: refs/heads/master 69b39c110 -> 287f52db0
DRILL-3155: Part 1: Move complex vectors logic in the correct package. Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/21de1381 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/21de1381 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/21de1381 Branch: refs/heads/master Commit: 21de138161a12ff0d01542950601b7944e9526ed Parents: 69b39c1 Author: Mehant Baid <[email protected]> Authored: Fri May 29 14:47:23 2015 -0700 Committer: Mehant Baid <[email protected]> Committed: Wed Jun 3 15:07:25 2015 -0700 ---------------------------------------------------------------------- .../codegen/templates/RepeatedValueVectors.java | 4 +- .../impl/flatten/FlattenRecordBatch.java | 2 +- .../physical/impl/flatten/FlattenTemplate.java | 2 +- .../exec/physical/impl/flatten/Flattener.java | 2 +- .../apache/drill/exec/store/VectorHolder.java | 107 ---------- .../columnreaders/FixedWidthRepeatedReader.java | 4 +- .../columnreaders/ParquetRecordReader.java | 2 +- .../drill/exec/vector/AllocationHelper.java | 3 +- .../exec/vector/BaseRepeatedValueVector.java | 207 ------------------ .../drill/exec/vector/ContainerVectorLike.java | 39 ---- .../vector/RepeatedFixedWidthVectorLike.java | 56 ----- .../drill/exec/vector/RepeatedMutator.java | 22 -- .../drill/exec/vector/RepeatedValueVector.java | 82 ------- .../vector/RepeatedVariableWidthVectorLike.java | 47 ---- .../vector/complex/BaseRepeatedValueVector.java | 213 +++++++++++++++++++ .../vector/complex/ContainerVectorLike.java | 43 ++++ .../complex/RepeatedFixedWidthVectorLike.java | 56 +++++ .../exec/vector/complex/RepeatedListVector.java | 5 - .../exec/vector/complex/RepeatedMapVector.java | 4 - .../vector/complex/RepeatedValueVector.java | 86 ++++++++ .../RepeatedVariableWidthVectorLike.java | 47 ++++ 21 files changed, 453 insertions(+), 580 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java index 7b2b78d..813c3f8 100644 --- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java +++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java @@ -19,9 +19,7 @@ import java.lang.Override; import org.apache.drill.exec.record.TransferPair; -import org.apache.drill.exec.vector.BaseRepeatedValueVector; -import org.apache.drill.exec.vector.BaseValueVector; -import org.apache.drill.exec.vector.RepeatedFixedWidthVectorLike; +import org.apache.drill.exec.vector.complex.BaseRepeatedValueVector; import org.mortbay.jetty.servlet.Holder; <@pp.dropOutputFile /> http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java index 00a78fd..9991404 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java @@ -50,7 +50,7 @@ import org.apache.drill.exec.record.TransferPair; import org.apache.drill.exec.record.TypedFieldId; import org.apache.drill.exec.record.VectorContainer; import org.apache.drill.exec.record.VectorWrapper; -import org.apache.drill.exec.vector.RepeatedValueVector; +import org.apache.drill.exec.vector.complex.RepeatedValueVector; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.complex.RepeatedMapVector; import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter; http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenTemplate.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenTemplate.java index b8d040c..de67b62 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenTemplate.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenTemplate.java @@ -31,7 +31,7 @@ import org.apache.drill.exec.record.selection.SelectionVector4; import com.google.common.collect.ImmutableList; -import org.apache.drill.exec.vector.RepeatedValueVector; +import org.apache.drill.exec.vector.complex.RepeatedValueVector; public abstract class FlattenTemplate implements Flattener { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FlattenTemplate.class); http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/Flattener.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/Flattener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/Flattener.java index 323bf43..92cf79d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/Flattener.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/Flattener.java @@ -24,7 +24,7 @@ import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.record.RecordBatch; import org.apache.drill.exec.record.TransferPair; -import org.apache.drill.exec.vector.RepeatedValueVector; +import org.apache.drill.exec.vector.complex.RepeatedValueVector; public interface Flattener { http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java deleted file mode 100644 index e602fd7..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store; - -import org.apache.drill.exec.vector.AllocationHelper; -import org.apache.drill.exec.vector.RepeatedFixedWidthVectorLike; -import org.apache.drill.exec.vector.RepeatedMutator; -import org.apache.drill.exec.vector.RepeatedVariableWidthVectorLike; -import org.apache.drill.exec.vector.ValueVector; - -public class VectorHolder { - private int count; - private int groupCount; - private int length; - private ValueVector vector; - private int currentLength; - private boolean repeated; - - - public VectorHolder(int length, ValueVector vector) { - this.length = length; - this.vector = vector; - if (vector instanceof RepeatedFixedWidthVectorLike || vector instanceof RepeatedVariableWidthVectorLike) { - repeated = true; - } - } - - public VectorHolder(ValueVector vector) { - this.length = vector.getValueCapacity(); - this.vector = vector; - if (vector instanceof RepeatedFixedWidthVectorLike || vector instanceof RepeatedVariableWidthVectorLike) { - repeated = true; - } - } - - public boolean isRepeated() { - return repeated; - } - - public ValueVector getValueVector() { - return vector; - } - - public void incAndCheckLength(int newLength) { - if (!hasEnoughSpace(newLength)) { - throw new BatchExceededException(length, vector.getBufferSize() + newLength); - } - - currentLength += newLength; - count += 1; - } - - public void setGroupCount(int groupCount) { - if (this.groupCount < groupCount) { - RepeatedMutator mutator = (RepeatedMutator) vector.getMutator(); - while (this.groupCount < groupCount) { - mutator.startNewGroup(++this.groupCount); - } - } - } - - public boolean hasEnoughSpace(int newLength) { - return length >= currentLength + newLength; - } - - public int getLength() { - return length; - } - - public void reset() { - currentLength = 0; - count = 0; - allocateNew(length); - } - - public void populateVectorLength() { - ValueVector.Mutator mutator = vector.getMutator(); - if (vector instanceof RepeatedFixedWidthVectorLike || vector instanceof RepeatedVariableWidthVectorLike) { - mutator.setValueCount(groupCount); - } else { - mutator.setValueCount(count); - } - } - - public void allocateNew(int valueLength) { - AllocationHelper.allocate(vector, valueLength, 10, 5); - } - - public void allocateNew(int valueLength, int repeatedPerTop) { - AllocationHelper.allocate(vector, valueLength, 10, repeatedPerTop); - } -} http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedWidthRepeatedReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedWidthRepeatedReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedWidthRepeatedReader.java index 2b929a4..d6e30ae 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedWidthRepeatedReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedWidthRepeatedReader.java @@ -21,10 +21,8 @@ import java.io.IOException; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.vector.BaseDataValueVector; -import org.apache.drill.exec.vector.RepeatedFixedWidthVectorLike; -import org.apache.drill.exec.vector.RepeatedValueVector; +import org.apache.drill.exec.vector.complex.RepeatedValueVector; import org.apache.drill.exec.vector.UInt4Vector; -import org.apache.drill.exec.vector.ValueVector; import parquet.column.ColumnDescriptor; import parquet.format.SchemaElement; http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java index 0cbd480..da6fbfb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java @@ -41,7 +41,7 @@ import org.apache.drill.exec.store.AbstractRecordReader; import org.apache.drill.exec.store.parquet.DirectCodecFactory; import org.apache.drill.exec.vector.AllocationHelper; import org.apache.drill.exec.vector.NullableIntVector; -import org.apache.drill.exec.vector.RepeatedValueVector; +import org.apache.drill.exec.vector.complex.RepeatedValueVector; import org.apache.drill.exec.vector.ValueVector; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java index eddefd0..e518042 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java @@ -17,7 +17,8 @@ */ package org.apache.drill.exec.vector; -import org.apache.drill.exec.memory.OutOfMemoryRuntimeException; +import org.apache.drill.exec.vector.complex.RepeatedFixedWidthVectorLike; +import org.apache.drill.exec.vector.complex.RepeatedVariableWidthVectorLike; public class AllocationHelper { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AllocationHelper.class); http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseRepeatedValueVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseRepeatedValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseRepeatedValueVector.java deleted file mode 100644 index d5a0d62..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseRepeatedValueVector.java +++ /dev/null @@ -1,207 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.vector; - -import java.util.Collections; -import java.util.Iterator; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ObjectArrays; -import io.netty.buffer.DrillBuf; -import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.common.types.Types; -import org.apache.drill.exec.exception.SchemaChangeRuntimeException; -import org.apache.drill.exec.expr.TypeHelper; -import org.apache.drill.exec.memory.BufferAllocator; -import org.apache.drill.exec.proto.UserBitShared; -import org.apache.drill.exec.record.MaterializedField; - -public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector { - - public final static ValueVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE; - public final static String OFFSETS_VECTOR_NAME = "offsets"; - public final static String DATA_VECTOR_NAME = "data"; - - private final static MaterializedField offsetsField = - MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(TypeProtos.MinorType.UINT4)); - - protected final UInt4Vector offsets; - protected ValueVector vector; - - protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator) { - this(field, allocator, DEFAULT_DATA_VECTOR); - } - - protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator, ValueVector vector) { - super(field, allocator); - this.offsets = new UInt4Vector(offsetsField, allocator); - this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null"); - } - - @Override - public boolean allocateNewSafe() { - if (!offsets.allocateNewSafe()) { - return false; - } - offsets.zeroVector(); - return vector.allocateNewSafe(); - } - - @Override - public UInt4Vector getOffsetVector() { - return offsets; - } - - @Override - public ValueVector getDataVector() { - return vector; - } - - @Override - public void setInitialCapacity(int numRecords) { - offsets.setInitialCapacity(numRecords + 1); - vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD); - } - - @Override - public int getValueCapacity() { - final int offsetValueCapacity = offsets.getValueCapacity() - 1; - if (vector == DEFAULT_DATA_VECTOR) { - return offsetValueCapacity; - } - return Math.min(vector.getValueCapacity(), offsetValueCapacity); - } - - @Override - protected UserBitShared.SerializedField.Builder getMetadataBuilder() { - return super.getMetadataBuilder() - .setGroupCount(getAccessor().getValueCount()) - .setValueCount(getAccessor().getInnerValueCount()) - .addChild(vector.getMetadata()); - } - - @Override - public int getBufferSize() { - if (getAccessor().getValueCount() == 0) { - return 0; - } - return offsets.getBufferSize() + vector.getBufferSize(); - } - - @Override - public Iterator<ValueVector> iterator() { - return Collections.singleton(getDataVector()).iterator(); - } - - @Override - public void clear() { - offsets.clear(); - vector.clear(); - super.clear(); - } - - @Override - public DrillBuf[] getBuffers(boolean clear) { - final DrillBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), DrillBuf.class); - if (clear) { - for (DrillBuf buffer:buffers) { - buffer.retain(); - } - clear(); - } - return buffers; - } - - /** - * Returns 1 if inner vector is explicitly set via #addOrGetVector else 0 - * - * @see {@link ContainerVectorLike#size} - */ - @Override - public int size() { - return vector == DEFAULT_DATA_VECTOR ? 0:1; - } - - @Override - public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) { - boolean created = false; - if (vector == DEFAULT_DATA_VECTOR && descriptor.getType().getMinorType() != TypeProtos.MinorType.LATE) { - final MaterializedField field = descriptor.withName(DATA_VECTOR_NAME).getField(); - vector = TypeHelper.getNewVector(field, allocator); - // returned vector must have the same field - assert field.equals(vector.getField()); - getField().addChild(field); - created = true; - } - - final TypeProtos.MajorType actual = vector.getField().getType(); - if (!actual.equals(descriptor.getType())) { - final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]", - descriptor.getType(), actual); - throw new SchemaChangeRuntimeException(msg); - } - - return new AddOrGetResult<>((T)vector, created); - } - - public abstract class BaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor { - - @Override - public int getValueCount() { - return Math.max(offsets.getAccessor().getValueCount() - 1, 0); - } - - @Override - public int getInnerValueCount() { - return vector.getAccessor().getValueCount(); - } - - @Override - public int getInnerValueCountAt(int index) { - return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index); - } - - @Override - public boolean isNull(int index) { - return false; - } - - @Override - public boolean isEmpty(int index) { - return false; - } - } - - public abstract class BaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator { - - @Override - public void startNewValue(int index) { - offsets.getMutator().setSafe(index+1, offsets.getAccessor().get(index)); - setValueCount(index+1); - } - - @Override - public void setValueCount(int valueCount) { - // TODO: populate offset end points - offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount+1); - final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount); - vector.getMutator().setValueCount(childValueCount); - } - } - -} http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ContainerVectorLike.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ContainerVectorLike.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ContainerVectorLike.java deleted file mode 100644 index 95e3365..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ContainerVectorLike.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.vector; - -/** - * A mix-in used for introducing container vector-like behaviour. - */ -public interface ContainerVectorLike { - - /** - * Creates and adds a child vector if none with the same name exists, else returns the vector instance. - * - * @param descriptor vector descriptor - * @return result of operation wrapping vector corresponding to the given descriptor and whether it's newly created - * @throws org.apache.drill.common.exceptions.DrillRuntimeException - * if schema change is not permissible between the given and existing data vector types. - */ - <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor); - - /** - * Returns the number of child vectors in this container vector-like instance. - */ - int size(); -} http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVectorLike.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVectorLike.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVectorLike.java deleted file mode 100644 index 450c673..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVectorLike.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.vector; - -import io.netty.buffer.DrillBuf; - -/** - * A {@link org.apache.drill.exec.vector.ValueVector} mix-in that can be used in conjunction with - * {@link org.apache.drill.exec.vector.RepeatedValueVector} subtypes. - */ -public interface RepeatedFixedWidthVectorLike { - /** - * Allocate a new memory space for this vector. Must be called prior to using the ValueVector. - * - * @param valueCount Number of separate repeating groupings. - * @param innerValueCount Number of supported values in the vector. - */ - public void allocateNew(int valueCount, int innerValueCount); - - /** - * Load the records in the provided buffer based on the given number of values. - * @param valueCount Number of separate repeating groupings. - * @param innerValueCount Number atomic values the buffer contains. - * @param buf Incoming buffer. - * @return The number of bytes of the buffer that were consumed. - */ - public int load(int valueCount, int innerValueCount, DrillBuf buf); - -// public interface RepeatedAccessor extends Accessor { -// public int getGroupCount(); -// public int getValueCount(); -// public int getGroupSizeAtIndex(int index); -// public ValueVector getAllChildValues(); -// } -// -// public interface RepeatedMutator extends Mutator { -// public void setValueCounts(int parentValueCount, int childValueCount); -// public void setRepetitionAtIndexSafe(int index, int repetitionCount); -// public BaseDataValueVector getDataVector(); -// } -} http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedMutator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedMutator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedMutator.java deleted file mode 100644 index 8e097e4..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedMutator.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.vector; - -public interface RepeatedMutator extends ValueVector.Mutator { - public void startNewGroup(int index); -} http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedValueVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedValueVector.java deleted file mode 100644 index 95a7252..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedValueVector.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.vector; - -/** - * An abstraction representing repeated value vectors. - * - * A repeated vector contains values that may either be flat or nested. A value consists of zero or more cells(inner values). - * Current design maintains data and offsets vectors. Each cell is stored in the data vector. Repeated vector - * uses the offset vector to determine the sequence of cells pertaining to an individual value. - * - */ -public interface RepeatedValueVector extends ValueVector, ContainerVectorLike { - - final static int DEFAULT_REPEAT_PER_RECORD = 5; - - /** - * Returns the underlying offset vector or null if none exists. - * - * TODO(DRILL-2995): eliminate exposing low-level interfaces. - */ - UInt4Vector getOffsetVector(); - - /** - * Returns the underlying data vector or null if none exists. - */ - ValueVector getDataVector(); - - @Override - RepeatedAccessor getAccessor(); - - @Override - RepeatedMutator getMutator(); - - interface RepeatedAccessor extends ValueVector.Accessor { - /** - * Returns total number of cells that vector contains. - * - * The result includes empty, null valued cells. - */ - int getInnerValueCount(); - - - /** - * Returns number of cells that the value at the given index contains. - */ - int getInnerValueCountAt(int index); - - /** - * Returns true if the value at the given index is empty, false otherwise. - * - * @param index value index - */ - boolean isEmpty(int index); - } - - interface RepeatedMutator extends ValueVector.Mutator { - /** - * Starts a new value that is a container of cells. - * - * @param index index of new value to start - */ - void startNewValue(int index); - - - } -} http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVectorLike.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVectorLike.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVectorLike.java deleted file mode 100644 index ac8589e..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVectorLike.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.vector; - -import io.netty.buffer.DrillBuf; - -public interface RepeatedVariableWidthVectorLike { - /** - * Allocate a new memory space for this vector. Must be called prior to using the ValueVector. - * - * @param totalBytes Desired size of the underlying data buffer. - * @param parentValueCount Number of separate repeating groupings. - * @param childValueCount Number of supported values in the vector. - */ - public void allocateNew(int totalBytes, int parentValueCount, int childValueCount); - - /** - * Provide the maximum amount of variable width bytes that can be stored int his vector. - * @return - */ - public int getByteCapacity(); - - /** - * Load the records in the provided buffer based on the given number of values. - * @param dataBytes The number of bytes associated with the data array. - * @param parentValueCount Number of separate repeating groupings. - * @param childValueCount Number of supported values in the vector. - * @param buf Incoming buffer. - * @return The number of bytes of the buffer that were consumed. - */ - public int load(int dataBytes, int parentValueCount, int childValueCount, DrillBuf buf); -} http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java new file mode 100644 index 0000000..88b44db --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java @@ -0,0 +1,213 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.vector.complex; + +import java.util.Collections; +import java.util.Iterator; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ObjectArrays; +import io.netty.buffer.DrillBuf; +import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.common.types.Types; +import org.apache.drill.exec.exception.SchemaChangeRuntimeException; +import org.apache.drill.exec.expr.TypeHelper; +import org.apache.drill.exec.memory.BufferAllocator; +import org.apache.drill.exec.proto.UserBitShared; +import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.vector.AddOrGetResult; +import org.apache.drill.exec.vector.BaseValueVector; +import org.apache.drill.exec.vector.UInt4Vector; +import org.apache.drill.exec.vector.ValueVector; +import org.apache.drill.exec.vector.VectorDescriptor; +import org.apache.drill.exec.vector.ZeroVector; + +public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector { + + public final static ValueVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE; + public final static String OFFSETS_VECTOR_NAME = "offsets"; + public final static String DATA_VECTOR_NAME = "data"; + + private final static MaterializedField offsetsField = + MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(TypeProtos.MinorType.UINT4)); + + protected final UInt4Vector offsets; + protected ValueVector vector; + + protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator) { + this(field, allocator, DEFAULT_DATA_VECTOR); + } + + protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator, ValueVector vector) { + super(field, allocator); + this.offsets = new UInt4Vector(offsetsField, allocator); + this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null"); + } + + @Override + public boolean allocateNewSafe() { + if (!offsets.allocateNewSafe()) { + return false; + } + offsets.zeroVector(); + return vector.allocateNewSafe(); + } + + @Override + public UInt4Vector getOffsetVector() { + return offsets; + } + + @Override + public ValueVector getDataVector() { + return vector; + } + + @Override + public void setInitialCapacity(int numRecords) { + offsets.setInitialCapacity(numRecords + 1); + vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD); + } + + @Override + public int getValueCapacity() { + final int offsetValueCapacity = offsets.getValueCapacity() - 1; + if (vector == DEFAULT_DATA_VECTOR) { + return offsetValueCapacity; + } + return Math.min(vector.getValueCapacity(), offsetValueCapacity); + } + + @Override + protected UserBitShared.SerializedField.Builder getMetadataBuilder() { + return super.getMetadataBuilder() + .setGroupCount(getAccessor().getValueCount()) + .setValueCount(getAccessor().getInnerValueCount()) + .addChild(vector.getMetadata()); + } + + @Override + public int getBufferSize() { + if (getAccessor().getValueCount() == 0) { + return 0; + } + return offsets.getBufferSize() + vector.getBufferSize(); + } + + @Override + public Iterator<ValueVector> iterator() { + return Collections.singleton(getDataVector()).iterator(); + } + + @Override + public void clear() { + offsets.clear(); + vector.clear(); + super.clear(); + } + + @Override + public DrillBuf[] getBuffers(boolean clear) { + final DrillBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), DrillBuf.class); + if (clear) { + for (DrillBuf buffer:buffers) { + buffer.retain(); + } + clear(); + } + return buffers; + } + + /** + * Returns 1 if inner vector is explicitly set via #addOrGetVector else 0 + * + * @see {@link ContainerVectorLike#size} + */ + @Override + public int size() { + return vector == DEFAULT_DATA_VECTOR ? 0:1; + } + + @Override + public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) { + boolean created = false; + if (vector == DEFAULT_DATA_VECTOR && descriptor.getType().getMinorType() != TypeProtos.MinorType.LATE) { + final MaterializedField field = descriptor.withName(DATA_VECTOR_NAME).getField(); + vector = TypeHelper.getNewVector(field, allocator); + // returned vector must have the same field + assert field.equals(vector.getField()); + getField().addChild(field); + created = true; + } + + final TypeProtos.MajorType actual = vector.getField().getType(); + if (!actual.equals(descriptor.getType())) { + final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]", + descriptor.getType(), actual); + throw new SchemaChangeRuntimeException(msg); + } + + return new AddOrGetResult<>((T)vector, created); + } + + public abstract class BaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor { + + @Override + public int getValueCount() { + return Math.max(offsets.getAccessor().getValueCount() - 1, 0); + } + + @Override + public int getInnerValueCount() { + return vector.getAccessor().getValueCount(); + } + + @Override + public int getInnerValueCountAt(int index) { + return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index); + } + + @Override + public boolean isNull(int index) { + return false; + } + + @Override + public boolean isEmpty(int index) { + return false; + } + } + + public abstract class BaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator { + + @Override + public void startNewValue(int index) { + offsets.getMutator().setSafe(index+1, offsets.getAccessor().get(index)); + setValueCount(index+1); + } + + @Override + public void setValueCount(int valueCount) { + // TODO: populate offset end points + offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount+1); + final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount); + vector.getMutator().setValueCount(childValueCount); + } + } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ContainerVectorLike.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ContainerVectorLike.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ContainerVectorLike.java new file mode 100644 index 0000000..999b47d --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ContainerVectorLike.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.vector.complex; + +import org.apache.drill.exec.vector.AddOrGetResult; +import org.apache.drill.exec.vector.ValueVector; +import org.apache.drill.exec.vector.VectorDescriptor; + +/** + * A mix-in used for introducing container vector-like behaviour. + */ +public interface ContainerVectorLike { + + /** + * Creates and adds a child vector if none with the same name exists, else returns the vector instance. + * + * @param descriptor vector descriptor + * @return result of operation wrapping vector corresponding to the given descriptor and whether it's newly created + * @throws org.apache.drill.common.exceptions.DrillRuntimeException + * if schema change is not permissible between the given and existing data vector types. + */ + <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor); + + /** + * Returns the number of child vectors in this container vector-like instance. + */ + int size(); +} http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java new file mode 100644 index 0000000..fb7ed2a --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.vector.complex; + +import io.netty.buffer.DrillBuf; + +/** + * A {@link org.apache.drill.exec.vector.ValueVector} mix-in that can be used in conjunction with + * {@link RepeatedValueVector} subtypes. + */ +public interface RepeatedFixedWidthVectorLike { + /** + * Allocate a new memory space for this vector. Must be called prior to using the ValueVector. + * + * @param valueCount Number of separate repeating groupings. + * @param innerValueCount Number of supported values in the vector. + */ + void allocateNew(int valueCount, int innerValueCount); + + /** + * Load the records in the provided buffer based on the given number of values. + * @param valueCount Number of separate repeating groupings. + * @param innerValueCount Number atomic values the buffer contains. + * @param buf Incoming buffer. + * @return The number of bytes of the buffer that were consumed. + */ + int load(int valueCount, int innerValueCount, DrillBuf buf); + +// public interface RepeatedAccessor extends Accessor { +// public int getGroupCount(); +// public int getValueCount(); +// public int getGroupSizeAtIndex(int index); +// public ValueVector getAllChildValues(); +// } +// +// public interface RepeatedMutator extends Mutator { +// public void setValueCounts(int parentValueCount, int childValueCount); +// public void setRepetitionAtIndexSafe(int index, int repetitionCount); +// public BaseDataValueVector getDataVector(); +// } +} http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java index a5553b2..f6d3d88 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java @@ -29,21 +29,16 @@ import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; -import org.apache.drill.exec.expr.TypeHelper; import org.apache.drill.exec.expr.holders.ComplexHolder; import org.apache.drill.exec.expr.holders.RepeatedListHolder; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.memory.OutOfMemoryRuntimeException; -import org.apache.drill.exec.proto.UserBitShared; import org.apache.drill.exec.proto.UserBitShared.SerializedField; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.TransferPair; import org.apache.drill.exec.util.JsonStringArrayList; import org.apache.drill.exec.vector.AddOrGetResult; -import org.apache.drill.exec.vector.BaseRepeatedValueVector; -import org.apache.drill.exec.vector.RepeatedFixedWidthVectorLike; import org.apache.drill.exec.util.CallBack; -import org.apache.drill.exec.vector.RepeatedValueVector; import org.apache.drill.exec.vector.UInt4Vector; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.VectorDescriptor; http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java index a97847b..2e12e55 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java @@ -43,10 +43,6 @@ import org.apache.drill.exec.util.CallBack; import org.apache.drill.exec.util.JsonStringArrayList; import org.apache.drill.exec.vector.AddOrGetResult; import org.apache.drill.exec.vector.AllocationHelper; -import org.apache.drill.exec.vector.BaseDataValueVector; -import org.apache.drill.exec.vector.BaseRepeatedValueVector; -import org.apache.drill.exec.vector.RepeatedFixedWidthVectorLike; -import org.apache.drill.exec.vector.RepeatedValueVector; import org.apache.drill.exec.vector.UInt4Vector; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.VectorDescriptor; http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java new file mode 100644 index 0000000..0fba292 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.vector.complex; + +import org.apache.drill.exec.vector.UInt4Vector; +import org.apache.drill.exec.vector.ValueVector; +import org.apache.drill.exec.vector.complex.ContainerVectorLike; + +/** + * An abstraction representing repeated value vectors. + * + * A repeated vector contains values that may either be flat or nested. A value consists of zero or more cells(inner values). + * Current design maintains data and offsets vectors. Each cell is stored in the data vector. Repeated vector + * uses the offset vector to determine the sequence of cells pertaining to an individual value. + * + */ +public interface RepeatedValueVector extends ValueVector, ContainerVectorLike { + + final static int DEFAULT_REPEAT_PER_RECORD = 5; + + /** + * Returns the underlying offset vector or null if none exists. + * + * TODO(DRILL-2995): eliminate exposing low-level interfaces. + */ + UInt4Vector getOffsetVector(); + + /** + * Returns the underlying data vector or null if none exists. + */ + ValueVector getDataVector(); + + @Override + RepeatedAccessor getAccessor(); + + @Override + RepeatedMutator getMutator(); + + interface RepeatedAccessor extends ValueVector.Accessor { + /** + * Returns total number of cells that vector contains. + * + * The result includes empty, null valued cells. + */ + int getInnerValueCount(); + + + /** + * Returns number of cells that the value at the given index contains. + */ + int getInnerValueCountAt(int index); + + /** + * Returns true if the value at the given index is empty, false otherwise. + * + * @param index value index + */ + boolean isEmpty(int index); + } + + interface RepeatedMutator extends ValueVector.Mutator { + /** + * Starts a new value that is a container of cells. + * + * @param index index of new value to start + */ + void startNewValue(int index); + + + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java new file mode 100644 index 0000000..c57143e --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.vector.complex; + +import io.netty.buffer.DrillBuf; + +public interface RepeatedVariableWidthVectorLike { + /** + * Allocate a new memory space for this vector. Must be called prior to using the ValueVector. + * + * @param totalBytes Desired size of the underlying data buffer. + * @param parentValueCount Number of separate repeating groupings. + * @param childValueCount Number of supported values in the vector. + */ + void allocateNew(int totalBytes, int parentValueCount, int childValueCount); + + /** + * Provide the maximum amount of variable width bytes that can be stored int his vector. + * @return + */ + int getByteCapacity(); + + /** + * Load the records in the provided buffer based on the given number of values. + * @param dataBytes The number of bytes associated with the data array. + * @param parentValueCount Number of separate repeating groupings. + * @param childValueCount Number of supported values in the vector. + * @param buf Incoming buffer. + * @return The number of bytes of the buffer that were consumed. + */ + int load(int dataBytes, int parentValueCount, int childValueCount, DrillBuf buf); +}
