DRILL-5517: Size-aware set methods in value vectors Please see DRILL-5517 for an explanation.
Also includes a workaround for DRILL-5529. Implements a setEmpties method for repeated and non-nullable variable-width types in support of the revised column accessors. Unit test included. Without the setEmpties call, the tests fail with vector corruption. With the call, things work properly. closes #840 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/92c9304f Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/92c9304f Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/92c9304f Branch: refs/heads/master Commit: 92c9304f77a515d8f6747514bd295613738f611e Parents: 6446e56 Author: Paul Rogers <[email protected]> Authored: Tue May 16 13:20:32 2017 -0700 Committer: Paul Rogers <[email protected]> Committed: Mon Jul 3 11:39:16 2017 -0700 ---------------------------------------------------------------------- .../exec/record/vector/TestValueVector.java | 7 +- .../apache/drill/vector/TestFillEmpties.java | 242 ++++++++ .../apache/drill/vector/TestVectorLimits.java | 484 ++++++++++++++++ .../org/apache/drill/vector/package-info.java | 22 + .../src/main/java/io/netty/buffer/DrillBuf.java | 19 + .../netty/buffer/UnsafeDirectLittleEndian.java | 75 ++- .../codegen/templates/FixedValueVectors.java | 577 +++++++++++++------ .../codegen/templates/NullableValueVectors.java | 163 +++++- .../codegen/templates/RepeatedValueVectors.java | 132 ++++- .../src/main/codegen/templates/UnionVector.java | 13 +- .../templates/VariableLengthVectors.java | 218 ++++++- .../drill/exec/vector/BaseDataValueVector.java | 21 +- .../drill/exec/vector/BaseValueVector.java | 13 +- .../org/apache/drill/exec/vector/BitVector.java | 85 ++- .../drill/exec/vector/FixedWidthVector.java | 5 +- .../apache/drill/exec/vector/ObjectVector.java | 39 +- .../apache/drill/exec/vector/ValueVector.java | 60 +- .../exec/vector/VectorOverflowException.java | 30 + .../apache/drill/exec/vector/VectorUtils.java | 63 ++ .../apache/drill/exec/vector/ZeroVector.java | 71 +-- .../vector/complex/BaseRepeatedValueVector.java | 18 +- .../drill/exec/vector/complex/ListVector.java | 8 +- .../drill/exec/vector/complex/MapVector.java | 14 +- .../exec/vector/complex/RepeatedListVector.java | 19 +- .../exec/vector/complex/RepeatedMapVector.java | 19 +- .../vector/complex/impl/AbstractBaseReader.java | 12 +- 26 files changed, 2059 insertions(+), 370 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java index f6044e1..5191f5f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -25,7 +25,6 @@ import java.nio.charset.Charset; import org.apache.drill.common.AutoCloseables; import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.types.MinorType; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.Types; import org.apache.drill.exec.ExecTest; @@ -47,7 +46,6 @@ import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.proto.UserBitShared; import org.apache.drill.exec.record.MaterializedField; -import org.apache.drill.exec.record.VectorWrapper; import org.apache.drill.exec.vector.BaseValueVector; import org.apache.drill.exec.vector.BitVector; import org.apache.drill.exec.vector.NullableFloat4Vector; @@ -97,6 +95,7 @@ public class TestValueVector extends ExecTest { @Test(expected = OversizedAllocationException.class) public void testFixedVectorReallocation() { final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE); + @SuppressWarnings("resource") final UInt4Vector vector = new UInt4Vector(field, allocator); // edge case 1: buffer size = max value capacity final int expectedValueCapacity = BaseValueVector.MAX_ALLOCATION_SIZE / 4; @@ -149,10 +148,10 @@ public class TestValueVector extends ExecTest { } } - @Test(expected = OversizedAllocationException.class) public void testVariableVectorReallocation() { final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE); + @SuppressWarnings("resource") final VarCharVector vector = new VarCharVector(field, allocator); // edge case 1: value count = MAX_VALUE_ALLOCATION final int expectedAllocationInBytes = BaseValueVector.MAX_ALLOCATION_SIZE; http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/java-exec/src/test/java/org/apache/drill/vector/TestFillEmpties.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/vector/TestFillEmpties.java b/exec/java-exec/src/test/java/org/apache/drill/vector/TestFillEmpties.java new file mode 100644 index 0000000..266bff2 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/vector/TestFillEmpties.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package org.apache.drill.vector; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.vector.BaseDataValueVector; +import org.apache.drill.exec.vector.IntVector; +import org.apache.drill.exec.vector.NullableVarCharVector; +import org.apache.drill.exec.vector.RepeatedVarCharVector; +import org.apache.drill.exec.vector.UInt4Vector; +import org.apache.drill.exec.vector.VarCharVector; +import org.apache.drill.exec.vector.VectorOverflowException; +import org.apache.drill.test.DrillTest; +import org.apache.drill.test.OperatorFixture; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import io.netty.buffer.DrillBuf; + +public class TestFillEmpties extends DrillTest { + + public static OperatorFixture fixture; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + fixture = OperatorFixture.builder().build(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + fixture.close(); + } + + // To be replaced by a test method in a separate commit. + + public static MaterializedField makeField(String name, MinorType dataType, DataMode mode) { + MajorType type = MajorType.newBuilder() + .setMinorType(dataType) + .setMode(mode) + .build(); + + return MaterializedField.create(name, type); + } + + @Test + public void testNullableVarChar() { + @SuppressWarnings("resource") + NullableVarCharVector vector = new NullableVarCharVector(makeField("a", MinorType.VARCHAR, DataMode.OPTIONAL), fixture.allocator()); + vector.allocateNew( ); + + // Create "foo", null, "bar", but omit the null. + + NullableVarCharVector.Mutator mutator = vector.getMutator(); + byte[] value = makeValue( "foo" ); + mutator.setSafe(0, value, 0, value.length); + + value = makeValue("bar"); + mutator.setSafe(2, value, 0, value.length); + + visualize(vector, 3); + verifyOffsets(vector.getValuesVector().getOffsetVector(), new int[] {0, 3, 3, 6}); + vector.close(); + } + + @Test + public void testVarChar() { + @SuppressWarnings("resource") + VarCharVector vector = new VarCharVector(makeField("a", MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator()); + vector.allocateNew( ); + + // Create "foo", null, "bar", but omit the null. + + VarCharVector.Mutator mutator = vector.getMutator(); + byte[] value = makeValue( "foo" ); + mutator.setSafe(0, value, 0, value.length); + + // Work around: test fails without this. But, only the new column writers + // call this method. + + try { + mutator.fillEmptiesBounded(0, 2); + } catch (VectorOverflowException e) { + fail(); + } + value = makeValue("bar"); + mutator.setSafe(2, value, 0, value.length); + + visualize(vector, 3); + verifyOffsets(vector.getOffsetVector(), new int[] {0, 3, 3, 6}); + vector.close(); + } + + @Test + public void testInt() { + @SuppressWarnings("resource") + IntVector vector = new IntVector(makeField("a", MinorType.INT, DataMode.REQUIRED), fixture.allocator()); + vector.allocateNew( ); + + // Create 1, 0, 2, but omit the 0. + + IntVector.Mutator mutator = vector.getMutator(); + mutator.setSafe(0, 1); + + mutator.setSafe(2, 3); + + visualize(vector, 3); + vector.close(); + } + + @Test + public void testRepeatedVarChar() { + @SuppressWarnings("resource") + RepeatedVarCharVector vector = new RepeatedVarCharVector(makeField("a", MinorType.VARCHAR, DataMode.REPEATED), fixture.allocator()); + vector.allocateNew( ); + + // Create "foo", null, "bar", but omit the null. + + RepeatedVarCharVector.Mutator mutator = vector.getMutator(); + mutator.startNewValue(0); + byte[] value = makeValue( "a" ); + mutator.addSafe(0, value, 0, value.length); + value = makeValue( "b" ); + mutator.addSafe(0, value, 0, value.length); + + // Work around: test fails without this. But, only the new column writers + // call this method. + + try { + mutator.fillEmptiesBounded(0, 2); + } catch (VectorOverflowException e) { + fail(); + } + mutator.startNewValue(2); + value = makeValue( "c" ); + mutator.addSafe(2, value, 0, value.length); + value = makeValue( "d" ); + mutator.addSafe(2, value, 0, value.length); + + visualize(vector, 3); + verifyOffsets(vector.getOffsetVector(), new int[] {0, 2, 2, 4}); + verifyOffsets(vector.getDataVector().getOffsetVector(), new int[] {0, 1, 2, 3, 4}); + vector.close(); + } + + private void visualize(RepeatedVarCharVector vector, int valueCount) { + visualize("Array Offsets", vector.getOffsetVector(), valueCount + 1); + visualize(vector.getDataVector(), vector.getOffsetVector().getAccessor().get(valueCount)); + } + + private void visualize(IntVector vector, int valueCount) { + System.out.print("Values: ["); + IntVector.Accessor accessor = vector.getAccessor(); + for (int i = 0; i < valueCount; i++) { + if (i > 0) { System.out.print(" "); } + System.out.print(accessor.get(i)); + } + System.out.println("]"); + } + + private void visualize(NullableVarCharVector vector, int valueCount) { + visualize("Is-set", vector.getAccessor(), valueCount); + visualize(vector.getValuesVector(), valueCount); + } + + private void visualize(VarCharVector vector, int valueCount) { + visualize("Offsets", vector.getOffsetVector(), valueCount + 1); + visualize("Data", vector.getBuffer(), vector.getOffsetVector().getAccessor().get(valueCount)); + } + + private void visualize(String label, UInt4Vector offsetVector, + int valueCount) { + System.out.print(label + ": ["); + UInt4Vector.Accessor accessor = offsetVector.getAccessor(); + for (int i = 0; i < valueCount; i++) { + if (i > 0) { System.out.print(" "); } + System.out.print(accessor.get(i)); + } + System.out.println("]"); + } + + private void visualize(String label, DrillBuf buffer, int valueCount) { + System.out.print(label + ": ["); + for (int i = 0; i < valueCount; i++) { + if (i > 0) { System.out.print(" "); } + System.out.print((char) buffer.getByte(i)); + } + System.out.println("]"); + } + + private void visualize(String label, BaseDataValueVector.BaseAccessor accessor, int valueCount) { + System.out.print(label + ": ["); + for (int i = 0; i < valueCount; i++) { + if (i > 0) { System.out.print(" "); } + System.out.print(accessor.isNull(i) ? 0 : 1); + } + System.out.println("]"); + } + + private void verifyOffsets(UInt4Vector offsetVector, int[] expected) { + UInt4Vector.Accessor accessor = offsetVector.getAccessor(); + for (int i = 0; i < expected.length; i++) { + assertEquals(expected[i], accessor.get(i)); + } + } + + /** + * Create a test value. Works only for the ASCII subset of characters, obviously. + * @param string + * @return + */ + private byte[] makeValue(String string) { + byte value[] = new byte[string.length()]; + for (int i = 0; i < value.length; i++) { + value[i] = (byte) string.charAt(i); + } + return value; + } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/java-exec/src/test/java/org/apache/drill/vector/TestVectorLimits.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/vector/TestVectorLimits.java b/exec/java-exec/src/test/java/org/apache/drill/vector/TestVectorLimits.java new file mode 100644 index 0000000..86bd206 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/vector/TestVectorLimits.java @@ -0,0 +1,484 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.vector; + +import static org.junit.Assert.*; + +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.vector.IntVector; +import org.apache.drill.exec.vector.NullableIntVector; +import org.apache.drill.exec.vector.NullableVarCharVector; +import org.apache.drill.exec.vector.RepeatedIntVector; +import org.apache.drill.exec.vector.ValueVector; +import org.apache.drill.exec.vector.VarCharVector; +import org.apache.drill.exec.vector.VectorOverflowException; +import org.apache.drill.test.DrillTest; +import org.apache.drill.test.OperatorFixture; +import org.bouncycastle.util.Arrays; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import io.netty.buffer.DrillBuf; + +/** + * Test the setScalar() methods in the various generated vector + * classes. Rather than test all 100+ vectors, we sample a few and + * rely on the fact that code is generated from a common template. + */ + +public class TestVectorLimits extends DrillTest { + + public static OperatorFixture fixture; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + fixture = OperatorFixture.builder().build(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + fixture.close(); + } + + /** + * Test a vector directly using the vector mutator to ensure + * that the <tt>setScalar</tt> method works for the maximum + * row count. + * <p> + * This test is a proxy for all the other fixed types, since all + * share the same code template. + */ + + @Test + public void testFixedVector() { + + // Create a non-nullable int vector: a typical fixed-size vector + + @SuppressWarnings("resource") + IntVector vector = new IntVector(makeField(MinorType.INT, DataMode.REQUIRED), fixture.allocator() ); + + // Sanity test of generated constants. + + assertTrue( IntVector.MAX_SCALAR_COUNT <= ValueVector.MAX_ROW_COUNT ); + assertEquals( 4, IntVector.VALUE_WIDTH ); + assertTrue( IntVector.NET_MAX_SCALAR_SIZE <= ValueVector.MAX_BUFFER_SIZE ); + + // Allocate a default size, small vector. Forces test of + // the auto-grow (setSafe()) aspect of setScalar(). + + vector.allocateNew( ); + + // Write to the vector until it complains. At that point, + // we should have written up to the static fixed value count + // (which is computed to stay below the capacity limit.) + + IntVector.Mutator mutator = vector.getMutator(); + for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) { + try { + mutator.setScalar(i, i); + } catch (VectorOverflowException e) { + assertEquals(IntVector.MAX_SCALAR_COUNT, i); + break; + } + } + + // The vector should be below the allocation limit. Since this + // is an int vector, in practice the size will be far below + // the overall limit (if the limit stays at 16 MB.) But, it should + // be at the type-specific limit since we filled up the vector. + + assertEquals(IntVector.NET_MAX_SCALAR_SIZE, vector.getBuffer().getActualMemoryConsumed()); + vector.close(); + } + + @Test + public void testNullableFixedVector() { + + @SuppressWarnings("resource") + NullableIntVector vector = new NullableIntVector(makeField(MinorType.INT, DataMode.OPTIONAL), fixture.allocator() ); + vector.allocateNew( ); + + NullableIntVector.Mutator mutator = vector.getMutator(); + for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) { + try { + mutator.setScalar(i, i); + } catch (VectorOverflowException e) { + assertEquals(IntVector.MAX_SCALAR_COUNT, i); + break; + } + } + + vector.close(); + } + + /** + * Repeated fixed vector. Using an int vector, each column array can hold + * 256 / 4 = 64 values. We write only 10. The vector becomes full when we + * exceed 64K items. + */ + + @Test + public void testRepeatedFixedVectorCountLimit() { + + @SuppressWarnings("resource") + RepeatedIntVector vector = new RepeatedIntVector(makeField(MinorType.INT, DataMode.REPEATED), fixture.allocator() ); + vector.allocateNew( ); + + RepeatedIntVector.Mutator mutator = vector.getMutator(); + top: + for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) { + if (! mutator.startNewValueBounded(i)) { + assertEquals(ValueVector.MAX_ROW_COUNT, i); + // Continue, let's check the addBounded method also + } + for (int j = 0; j < 10; j++) { + try { + mutator.addEntry(i, i * 100 + j); + } catch (VectorOverflowException e) { + assertEquals(ValueVector.MAX_ROW_COUNT, i); + mutator.setValueCount(i); + break top; + } + } + } + + vector.close(); + } + + /** + * Repeated fixed vector. Using an int vector, each column array can hold + * 256 / 4 = 64 values. We write 100. The vector becomes full when we + * exceed the 16 MB size limit. + */ + + @Test + public void testRepeatedFixedVectorBufferLimit() { + + @SuppressWarnings("resource") + RepeatedIntVector vector = new RepeatedIntVector(makeField(MinorType.INT, DataMode.REPEATED), fixture.allocator() ); + vector.allocateNew( ); + + RepeatedIntVector.Mutator mutator = vector.getMutator(); + top: + for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) { + // We'll never hit the value count limit + assertTrue(mutator.startNewValueBounded(i)); + for (int j = 0; j < 100; j++) { + try { + mutator.addEntry(i, i * 100 + j); + } catch (VectorOverflowException e) { + // We should have hit the buffer limit before the value limit. + assertTrue(i < ValueVector.MAX_ROW_COUNT); + mutator.setValueCount(i); + break top; + } + } + } + + vector.close(); + } + + // To be replaced by a test method in a separate commit. + + public static MaterializedField makeField(MinorType dataType, DataMode mode) { + MajorType type = MajorType.newBuilder() + .setMinorType(dataType) + .setMode(mode) + .build(); + + return MaterializedField.create("foo", type); + } + + /** + * Baseline test for a variable-width vector using <tt>setSafe</tt> and + * loading the vector up to the maximum size. Doing so will cause the vector + * to have a buffer that exceeds the maximum size, demonstrating the + * need for <tt>setScalar()</tt>. + */ + + @Test + public void variableVectorBaseline() { + + // Create a non-nullable VarChar vector: a typical variable-size vector + + @SuppressWarnings("resource") + VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator() ); + vector.allocateNew( ); + + // A 16 MB value can hold 64K values of up to 256 bytes each. + // To force a size overflow, write values much larger. + // Write the maximum number of values which will silently + // allow the vector to grow beyond the critical size of 16 MB. + // Doing this in production would lead to memory fragmentation. + // So, this is what the setScalar() method assures we don't do. + + byte dummyValue[] = new byte[512]; + Arrays.fill(dummyValue, (byte) 'X'); + VarCharVector.Mutator mutator = vector.getMutator(); + for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) { + mutator.setSafe(i, dummyValue, 0, dummyValue.length); + } + + // The vector should be above the allocation limit. + // This is why code must migrate to the setScalar() call + // away from the setSafe() call. + + assertTrue(ValueVector.MAX_BUFFER_SIZE < vector.getBuffer().getActualMemoryConsumed()); + vector.close(); + } + + /** + * Test a vector directly using the vector mutator to ensure + * that the <tt>setScalar</tt> method works for the maximum + * vector size. + */ + + @Test + public void testWideVariableVector() { + + @SuppressWarnings("resource") + VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator() ); + vector.allocateNew( ); + + // A 16 MB value can hold 64K values of up to 256 bytes each. + // To force a size overflow, write values much larger. + // Write to the vector until it complains. At that point, + // we should have written up to the maximum buffer size. + + byte dummyValue[] = makeVarCharValue(512); + VarCharVector.Mutator mutator = vector.getMutator(); + int count = 0; + for ( ; count < 2 * ValueVector.MAX_ROW_COUNT; count++) { + try { + mutator.setScalar(count, dummyValue, 0, dummyValue.length); + } catch (VectorOverflowException e) { + break; + } + } + + // The vector should be at the allocation limit. If it wasn't, we + // should have grown it to hold more data. The value count will + // be below the maximum. + + mutator.setValueCount(count); + assertEquals(ValueVector.MAX_BUFFER_SIZE, vector.getBuffer().getActualMemoryConsumed()); + assertTrue(count < ValueVector.MAX_ROW_COUNT); + vector.close(); + } + + private byte[] makeVarCharValue(int n) { + byte dummyValue[] = new byte[n]; + Arrays.fill(dummyValue, (byte) 'X'); + return dummyValue; + } + + @Test + public void testNullableWideVariableVector() { + + @SuppressWarnings("resource") + NullableVarCharVector vector = new NullableVarCharVector(makeField(MinorType.VARCHAR, DataMode.OPTIONAL), fixture.allocator() ); + vector.allocateNew( ); + + byte dummyValue[] = makeVarCharValue(512); + NullableVarCharVector.Mutator mutator = vector.getMutator(); + int count = 0; + for ( ; count < 2 * ValueVector.MAX_ROW_COUNT; count++) { + try { + mutator.setScalar(count, dummyValue, 0, dummyValue.length); + } catch (VectorOverflowException e) { + break; + } + } + + mutator.setValueCount(count); + assertEquals(ValueVector.MAX_BUFFER_SIZE, vector.getValuesVector().getBuffer().getActualMemoryConsumed()); + assertTrue(count < ValueVector.MAX_ROW_COUNT); + vector.close(); + } + + /** + * Test a vector directly using the vector mutator to ensure + * that the <tt>setScalar</tt> method works for the maximum + * value count. + */ + + @Test + public void testNarrowVariableVector() { + + @SuppressWarnings("resource") + VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator() ); + vector.allocateNew( ); + + // Write small values that fit into 16 MB. We should stop writing + // when we reach the value count limit. + + byte dummyValue[] = makeVarCharValue(254); + VarCharVector.Mutator mutator = vector.getMutator(); + int count = 0; + for (; count < 2 * ValueVector.MAX_ROW_COUNT; count++) { + try { + mutator.setScalar(count, dummyValue, 0, dummyValue.length); + } catch (VectorOverflowException e) { + break; + } + } + + // Buffer size should be at or below the maximum, with count + // at the maximum. + + mutator.setValueCount(count); + assertTrue(vector.getBuffer().getActualMemoryConsumed() <= ValueVector.MAX_BUFFER_SIZE); + assertEquals(ValueVector.MAX_ROW_COUNT, count); + vector.close(); + } + + /** + * Test a vector directly using the vector mutator to ensure + * that the <tt>setScalar</tt> method works for the maximum + * value count. Uses a DrillBuf as input. + */ + + @Test + public void testDirectVariableVector() { + + @SuppressWarnings("resource") + VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator() ); + vector.allocateNew( ); + + // Repeat the big-value test, but with data coming from a DrillBuf + // (direct memory) rather than a heap array. + + @SuppressWarnings("resource") + DrillBuf drillBuf = makeVarCharValueDirect(260); + VarCharVector.Mutator mutator = vector.getMutator(); + int count = 0; + for (; count < 2 * ValueVector.MAX_ROW_COUNT; count++) { + try { + mutator.setScalar(count, drillBuf, 0, 260); + } catch (VectorOverflowException e) { + break; + } + } + drillBuf.close(); + + // Again, vector should be at the size limit, count below the + // value limit. + + mutator.setValueCount(count); + assertEquals(ValueVector.MAX_BUFFER_SIZE, vector.getBuffer().getActualMemoryConsumed()); + assertTrue(count < ValueVector.MAX_ROW_COUNT); + vector.close(); + } + + private DrillBuf makeVarCharValueDirect(int n) { + byte dummyValue[] = makeVarCharValue(n); + DrillBuf drillBuf = fixture.allocator().buffer(dummyValue.length); + drillBuf.setBytes(0, dummyValue); + return drillBuf; + } + + @Test + public void testDirectNullableVariableVector() { + + @SuppressWarnings("resource") + NullableVarCharVector vector = new NullableVarCharVector(makeField(MinorType.VARCHAR, DataMode.OPTIONAL), fixture.allocator() ); + vector.allocateNew( ); + + @SuppressWarnings("resource") + DrillBuf drillBuf = makeVarCharValueDirect(260); + NullableVarCharVector.Mutator mutator = vector.getMutator(); + int count = 0; + for (; count < 2 * ValueVector.MAX_ROW_COUNT; count++) { + try { + mutator.setScalar(count, drillBuf, 0, 260); + } catch (VectorOverflowException e) { + break; + } + } + drillBuf.close(); + + mutator.setValueCount(count); + assertEquals(ValueVector.MAX_BUFFER_SIZE, vector.getValuesVector().getBuffer().getActualMemoryConsumed()); + assertTrue(count < ValueVector.MAX_ROW_COUNT); + vector.close(); + } + + public static void main(String args[]) { + try { + setUpBeforeClass(); + new TestVectorLimits().performanceTest(); + tearDownAfterClass(); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + private void performanceTest() { + @SuppressWarnings("resource") + VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.OPTIONAL), fixture.allocator() ); + byte value[] = makeVarCharValue(1); + int warmCount = 100; + timeSetSafe(vector, value, warmCount); + runSetBounded(vector, value, warmCount); + int runCount = 1000; + timeSetSafe(vector, value, runCount); + runSetBounded(vector, value, runCount); + timeSetSafe(vector, value, runCount); + vector.close(); + } + + private void timeSetSafe(VarCharVector vector, byte[] value, int iterCount) { + long start = System.currentTimeMillis(); + for (int i = 0; i < iterCount; i++) { + vector.clear(); + vector.allocateNew( ); + + VarCharVector.Mutator mutator = vector.getMutator(); + for (int j = 0; j < ValueVector.MAX_ROW_COUNT; j++) { + mutator.setSafe(j, value, 0, value.length); + } + } + long elapsed = System.currentTimeMillis() - start; + System.out.println( iterCount + " runs of setSafe: " + elapsed + " ms." ); + } + + private void runSetBounded(VarCharVector vector, byte[] value, int iterCount) { + long start = System.currentTimeMillis(); + for (int i = 0; i < iterCount; i++) { + vector.clear(); + vector.allocateNew( ); + + VarCharVector.Mutator mutator = vector.getMutator(); + int posn = 0; + for (;;) { + try { + mutator.setScalar(posn++, value, 0, value.length); + } catch (VectorOverflowException e) { + break; + } + } + } + long elapsed = System.currentTimeMillis() - start; + System.out.println( iterCount + " runs of setScalar: " + elapsed + " ms." ); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/java-exec/src/test/java/org/apache/drill/vector/package-info.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/vector/package-info.java b/exec/java-exec/src/test/java/org/apache/drill/vector/package-info.java new file mode 100644 index 0000000..c858814 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/vector/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Tests for value vectors. Is in this module to allow use of + * the test tools which are available only in this module. + */ +package org.apache.drill.vector; http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java ---------------------------------------------------------------------- diff --git a/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java index 3793f25..5139086 100644 --- a/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java +++ b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java @@ -748,6 +748,25 @@ public final class DrillBuf extends AbstractByteBuf implements AutoCloseable { return this; } + // Clone of UDLE's setBytes(), but with bounds checking done as a boolean, + // not assertion. + + public boolean setBytesBounded(int index, byte[] src, int srcIndex, int length) { + // Must do here because Drill's UDLE is not ref counted. + // Done as an assert to avoid production overhead: if this is going + // to fail, it will do so spectacularly in tests, due to a programming error. + assert refCnt() > 0; + return udle.setBytesBounded(index, src, srcIndex, length); + } + + // As above, but for direct memory. + + public boolean setBytesBounded(int index, DrillBuf src, int srcIndex, int length) { + // See above. + assert refCnt() > 0; + return udle.setBytesBounded(index, src.udle, srcIndex, length); + } + @Override public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) { udle.setBytes(index + offset, src, srcIndex, length); http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java ---------------------------------------------------------------------- diff --git a/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java index 6495d5d..c91944f 100644 --- a/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java +++ b/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java @@ -18,15 +18,17 @@ package io.netty.buffer; -import io.netty.util.internal.PlatformDependent; - import java.nio.ByteOrder; import java.util.concurrent.atomic.AtomicLong; +import io.netty.util.internal.PlatformDependent; + /** - * The underlying class we use for little-endian access to memory. Is used underneath DrillBufs to abstract away the - * Netty classes and underlying Netty memory management. + * The underlying class we use for little-endian access to memory. Is used + * underneath DrillBufs to abstract away the Netty classes and underlying Netty + * memory management. */ + public final class UnsafeDirectLittleEndian extends WrappedByteBuf { private static final boolean NATIVE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN; private static final AtomicLong ID_GENERATOR = new AtomicLong(0); @@ -49,7 +51,6 @@ public final class UnsafeDirectLittleEndian extends WrappedByteBuf { UnsafeDirectLittleEndian(PooledUnsafeDirectByteBuf buf, AtomicLong bufferCount, AtomicLong bufferSize) { this(buf, true, bufferCount, bufferSize); - } private UnsafeDirectLittleEndian(AbstractByteBuf buf, boolean fake, AtomicLong bufferCount, AtomicLong bufferSize) { @@ -67,21 +68,21 @@ public final class UnsafeDirectLittleEndian extends WrappedByteBuf { this.wrapped = buf; this.memoryAddress = buf.memoryAddress(); } - private long addr(int index) { - return memoryAddress + index; - } - @Override - public long getLong(int index) { -// wrapped.checkIndex(index, 8); - long v = PlatformDependent.getLong(addr(index)); - return v; - } + private long addr(int index) { + return memoryAddress + index; + } - @Override - public float getFloat(int index) { - return Float.intBitsToFloat(getInt(index)); - } + @Override + public long getLong(int index) { +// wrapped.checkIndex(index, 8); + return PlatformDependent.getLong(addr(index)); + } + + @Override + public float getFloat(int index) { + return Float.intBitsToFloat(getInt(index)); + } @Override public ByteBuf slice() { @@ -174,6 +175,43 @@ public final class UnsafeDirectLittleEndian extends WrappedByteBuf { return this; } + // Clone of the super class checkIndex, but this version returns a boolean rather + // than throwing an exception. + + protected boolean hasCapacity(int index, int fieldLength) { + assert fieldLength >= 0; + return (! (index < 0 || index > capacity() - fieldLength)); + } + + /** + * Write bytes into the buffer at the given index, if space is available. + * @param index location to write + * @param src bytes to write + * @param srcIndex start of data in the source array + * @param length length of the data to write + * @return true if the value was written, false if the value was not + * written because the value would overflow the buffer + */ + + public boolean setBytesBounded(int index, byte[] src, int srcIndex, int length) { + if (! hasCapacity(index, length)) { + return false; + } + PlatformDependent.copyMemory(src, srcIndex, addr(index), length); + return true; + } + + // Version of the super class setBytes(), but with bounds checking done as a boolean, + // not assertion. This version requires a direct source buffer. + + public boolean setBytesBounded(int index, UnsafeDirectLittleEndian src, int srcIndex, int length) { + if (! hasCapacity(index, length)) { + return false; + } + PlatformDependent.copyMemory(src.memoryAddress() + srcIndex, addr(index), length); + return true; + } + @Override public ByteBuf writeShort(int value) { wrapped.ensureWritable(2); @@ -266,5 +304,4 @@ public final class UnsafeDirectLittleEndian extends WrappedByteBuf { assert isAssertEnabled = true; ASSERT_ENABLED = isAssertEnabled; } - } http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/codegen/templates/FixedValueVectors.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java b/exec/vector/src/main/codegen/templates/FixedValueVectors.java index 23188ce..1e83a4f 100644 --- a/exec/vector/src/main/codegen/templates/FixedValueVectors.java +++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java @@ -31,22 +31,60 @@ package org.apache.drill.exec.vector; import org.apache.drill.exec.util.DecimalUtility; /** - * ${minor.class} implements a vector of fixed width values. Elements in the vector are accessed - * by position, starting from the logical start of the vector. Values should be pushed onto the - * vector sequentially, but may be randomly accessed. - * The width of each element is ${type.width} byte(s) - * The equivalent Java primitive is '${minor.javaType!type.javaType}' + * ${minor.class} implements a vector of fixed width values. Elements in the vector are accessed + * by position, starting from the logical start of the vector. Values should be pushed onto the + * vector sequentially, but may be accessed randomly. + * <ul> + * <li>The width of each element is {@link #VALUE_WIDTH} (= ${type.width}) byte<#if type.width != 1>s</#if>.</li> + * <li>The equivalent Java primitive is '${minor.javaType!type.javaType}'.</li> + * </ul> * * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ -public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{ +public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class); + /** + * Width of each fixed-width value. + */ + + public static final int VALUE_WIDTH = ${type.width}; + + /** + * Maximum number of values that this fixed-width vector can hold + * and stay below the maximum vector size limit. This is the limit + * enforced when the vector is used to hold values in a repeated + * vector. + */ + + public static final int MAX_VALUE_COUNT = MAX_BUFFER_SIZE / VALUE_WIDTH; + + /** + * Maximum number of values that this fixed-width vector can hold + * and stay below the maximum vector size limit and/or stay below + * the maximum row count. This is the limit enforced when the + * vector is used to hold scalar (required or nullable) values. + * <p> + * Note: <tt>MAX_ROW_COUNT</tt> is defined in the parent <tt>ValueVector</tt> + * class as the maximum number of rows in a record batch (64K). Use this + * in place of the <tt>Character.MAX_SIZE</tt> value previously used. + */ + + public static final int MAX_SCALAR_COUNT = Math.min(MAX_ROW_COUNT, MAX_VALUE_COUNT); + + /** + * Actual maximum vector size, in bytes, given the number of fixed-width + * values that either fit in the maximum overall vector size, or that + * is no larger than the maximum vector item count. + */ + + public static final int NET_MAX_SCALAR_SIZE = VALUE_WIDTH * MAX_SCALAR_COUNT; + private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this); private final Accessor accessor = new Accessor(); private final Mutator mutator = new Mutator(); - private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION * ${type.width}; + private int allocationSizeInBytes = Math.min(INITIAL_VALUE_ALLOCATION * VALUE_WIDTH, MAX_BUFFER_SIZE); private int allocationMonitor = 0; public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) { @@ -54,45 +92,41 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } @Override - public FieldReader getReader(){ - return reader; - } + public FieldReader getReader() { return reader; } @Override public int getBufferSizeFor(final int valueCount) { if (valueCount == 0) { return 0; } - return valueCount * ${type.width}; + return valueCount * VALUE_WIDTH; } @Override public int getValueCapacity(){ - return data.capacity() / ${type.width}; + return data.capacity() / VALUE_WIDTH; } @Override - public Accessor getAccessor(){ - return accessor; - } + public Accessor getAccessor() { return accessor; } @Override - public Mutator getMutator(){ - return mutator; - } + public Mutator getMutator() { return mutator; } @Override public void setInitialCapacity(final int valueCount) { - final long size = 1L * valueCount * ${type.width}; + final long size = (long) valueCount * VALUE_WIDTH; + // TODO: Replace this with MAX_BUFFER_SIZE once all + // code is aware of the maximum vector size. if (size > MAX_ALLOCATION_SIZE) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size"); } - allocationSizeInBytes = (int)size; + allocationSizeInBytes = (int) size; } @Override public void allocateNew() { - if(!allocateNewSafe()){ + if (!allocateNewSafe()){ throw new OutOfMemoryException("Failure while allocating buffer."); } } @@ -123,11 +157,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F * Note that the maximum number of values a vector can allocate is Integer.MAX_VALUE / value width. * * @param valueCount - * @throws org.apache.drill.exec.memory.OutOfMemoryException if it can't allocate the new buffer + * @throws OutOfMemoryException if it can't allocate the new buffer */ @Override public void allocateNew(final int valueCount) { - allocateBytes(valueCount * ${type.width}); + allocateBytes(valueCount * VALUE_WIDTH); } @Override @@ -139,6 +173,8 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } private void allocateBytes(final long size) { + // TODO: Replace this with MAX_BUFFER_SIZE once all + // code is aware of the maximum vector size. if (size > MAX_ALLOCATION_SIZE) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size"); } @@ -150,13 +186,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F allocationSizeInBytes = curSize; } -/** - * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one - * - * @throws org.apache.drill.exec.memory.OutOfMemoryException if it can't allocate the new buffer - */ + /** + * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one + * + * @throws org.apache.drill.exec.memory.OutOfMemoryException if it can't allocate the new buffer + */ public void reAlloc() { final long newAllocationSize = allocationSizeInBytes * 2L; + // TODO: Replace this with MAX_BUFFER_SIZE once all + // code is aware of the maximum vector size. if (newAllocationSize > MAX_ALLOCATION_SIZE) { throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached."); } @@ -185,7 +223,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F Preconditions.checkArgument(this.field.getPath().equals(metadata.getNamePart().getName()), "The field %s doesn't match the provided metadata %s.", this.field, metadata); final int actualLength = metadata.getBufferLength(); final int valueCount = metadata.getValueCount(); - final int expectedLength = valueCount * ${type.width}; + final int expectedLength = valueCount * VALUE_WIDTH; assert actualLength == expectedLength : String.format("Expected to load %d bytes but actually loaded %d bytes", expectedLength, actualLength); clear(); @@ -220,8 +258,8 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target) { - final int startPoint = startIndex * ${type.width}; - final int sliceLength = length * ${type.width}; + final int startPoint = startIndex * VALUE_WIDTH; + final int sliceLength = length * VALUE_WIDTH; target.clear(); target.data = data.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer; target.data.writerIndex(sliceLength); @@ -229,7 +267,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F @Override public int getPayloadByteCount() { - return getAccessor().getValueCount() * ${type.width}; + return getAccessor().getValueCount() * VALUE_WIDTH; } private class TransferImpl implements TransferPair{ @@ -266,10 +304,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){ <#if (type.width > 8)> - from.data.getBytes(fromIndex * ${type.width}, data, thisIndex * ${type.width}, ${type.width}); + from.data.getBytes(fromIndex * VALUE_WIDTH, data, thisIndex * VALUE_WIDTH, VALUE_WIDTH); <#else> <#-- type.width <= 8 --> - data.set${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width}, - from.data.get${(minor.javaType!type.javaType)?cap_first}(fromIndex * ${type.width}) + data.set${(minor.javaType!type.javaType)?cap_first}(thisIndex * VALUE_WIDTH, + from.data.get${(minor.javaType!type.javaType)?cap_first}(fromIndex * VALUE_WIDTH) ); </#if> <#-- type.width --> } @@ -281,6 +319,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F copyFrom(fromIndex, thisIndex, from); } + @Override + public void copyEntry(int toIndex, ValueVector from, int fromIndex) { + ((${minor.class}Vector) from).data.getBytes(fromIndex * ${type.width}, data, toIndex * ${type.width}, ${type.width}); + } + public void decrementAllocationMonitor() { if (allocationMonitor > 0) { allocationMonitor = 0; @@ -295,7 +338,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F public final class Accessor extends BaseDataValueVector.BaseAccessor { @Override public int getValueCount() { - return data.writerIndex() / ${type.width}; + return data.writerIndex() / VALUE_WIDTH; } @Override @@ -305,20 +348,19 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F <#if (type.width > 8)> public ${minor.javaType!type.javaType} get(int index) { - return data.slice(index * ${type.width}, ${type.width}); + return data.slice(index * VALUE_WIDTH, VALUE_WIDTH); } <#if (minor.class == "Interval")> public void get(int index, ${minor.class}Holder holder){ - - final int offsetIndex = index * ${type.width}; + final int offsetIndex = index * VALUE_WIDTH; holder.months = data.getInt(offsetIndex); holder.days = data.getInt(offsetIndex + ${minor.daysOffset}); holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset}); } public void get(int index, Nullable${minor.class}Holder holder){ - final int offsetIndex = index * ${type.width}; + final int offsetIndex = index * VALUE_WIDTH; holder.isSet = 1; holder.months = data.getInt(offsetIndex); holder.days = data.getInt(offsetIndex + ${minor.daysOffset}); @@ -327,7 +369,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F @Override public ${friendlyType} getObject(int index) { - final int offsetIndex = index * ${type.width}; + final int offsetIndex = index * VALUE_WIDTH; final int months = data.getInt(offsetIndex); final int days = data.getInt(offsetIndex + ${minor.daysOffset}); final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset}); @@ -337,7 +379,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F public StringBuilder getAsStringBuilder(int index) { - final int offsetIndex = index * ${type.width}; + final int offsetIndex = index * VALUE_WIDTH; int months = data.getInt(offsetIndex); final int days = data.getInt(offsetIndex + ${minor.daysOffset}); @@ -372,14 +414,13 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F <#elseif (minor.class == "IntervalDay")> public void get(int index, ${minor.class}Holder holder){ - - final int offsetIndex = index * ${type.width}; + final int offsetIndex = index * VALUE_WIDTH; holder.days = data.getInt(offsetIndex); holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset}); } public void get(int index, Nullable${minor.class}Holder holder){ - final int offsetIndex = index * ${type.width}; + final int offsetIndex = index * VALUE_WIDTH; holder.isSet = 1; holder.days = data.getInt(offsetIndex); holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset}); @@ -387,7 +428,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F @Override public ${friendlyType} getObject(int index) { - final int offsetIndex = index * ${type.width}; + final int offsetIndex = index * VALUE_WIDTH; final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset}); final int days = data.getInt(offsetIndex); final Period p = new Period(); @@ -395,7 +436,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } public StringBuilder getAsStringBuilder(int index) { - final int offsetIndex = index * ${type.width}; + final int offsetIndex = index * VALUE_WIDTH; int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset}); final int days = data.getInt(offsetIndex); @@ -419,10 +460,9 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F append(millis)); } - <#elseif (minor.class == "Decimal28Sparse") || (minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")> - + <#elseif minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense"> public void get(int index, ${minor.class}Holder holder) { - holder.start = index * ${type.width}; + holder.start = index * VALUE_WIDTH; holder.buffer = data; holder.scale = getField().getScale(); holder.precision = getField().getPrecision(); @@ -430,7 +470,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F public void get(int index, Nullable${minor.class}Holder holder) { holder.isSet = 1; - holder.start = index * ${type.width}; + holder.start = index * VALUE_WIDTH; holder.buffer = data; holder.scale = getField().getScale(); holder.precision = getField().getPrecision(); @@ -440,65 +480,61 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F public ${friendlyType} getObject(int index) { <#if (minor.class == "Decimal28Sparse") || (minor.class == "Decimal38Sparse")> // Get the BigDecimal object - return DecimalUtility.getBigDecimalFromSparse(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale()); + return DecimalUtility.getBigDecimalFromSparse(data, index * VALUE_WIDTH, ${minor.nDecimalDigits}, getField().getScale()); <#else> - return DecimalUtility.getBigDecimalFromDense(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale(), ${minor.maxPrecisionDigits}, ${type.width}); + return DecimalUtility.getBigDecimalFromDense(data, index * VALUE_WIDTH, ${minor.nDecimalDigits}, getField().getScale(), ${minor.maxPrecisionDigits}, VALUE_WIDTH); </#if> } <#else> public void get(int index, ${minor.class}Holder holder){ holder.buffer = data; - holder.start = index * ${type.width}; + holder.start = index * VALUE_WIDTH; } public void get(int index, Nullable${minor.class}Holder holder){ holder.isSet = 1; holder.buffer = data; - holder.start = index * ${type.width}; + holder.start = index * VALUE_WIDTH; } @Override public ${friendlyType} getObject(int index) { - return data.slice(index * ${type.width}, ${type.width}) + return data.slice(index * VALUE_WIDTH, VALUE_WIDTH) } </#if> <#else> <#-- type.width <= 8 --> - public ${minor.javaType!type.javaType} get(int index) { - return data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); + return data.get${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH); } <#if type.width == 4> public long getTwoAsLong(int index) { - return data.getLong(index * ${type.width}); + return data.getLong(index * VALUE_WIDTH); } </#if> - <#if minor.class == "Date"> @Override public ${friendlyType} getObject(int index) { - org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); - date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); - return date; + org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); + date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + return date; } <#elseif minor.class == "TimeStamp"> @Override public ${friendlyType} getObject(int index) { - org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); - date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); - return date; + org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); + date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + return date; } <#elseif minor.class == "IntervalYear"> @Override public ${friendlyType} getObject(int index) { - final int value = get(index); - final int years = (value / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); final int months = (value % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); final Period p = new Period(); @@ -523,18 +559,16 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F <#elseif minor.class == "Time"> @Override public DateTime getObject(int index) { - - org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); - time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); - return time; + org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); + time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + return time; } <#elseif minor.class == "Decimal9" || minor.class == "Decimal18"> @Override public ${friendlyType} getObject(int index) { - - final BigInteger value = BigInteger.valueOf(((${type.boxedType})get(index)).${type.javaType}Value()); - return new BigDecimal(value, getField().getScale()); + final BigInteger value = BigInteger.valueOf(((${type.boxedType})get(index)).${type.javaType}Value()); + return new BigDecimal(value, getField().getScale()); } <#else> @@ -546,191 +580,298 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F public ${minor.javaType!type.javaType} getPrimitiveObject(int index) { return get(index); } - </#if> + </#if> public void get(int index, ${minor.class}Holder holder){ <#if minor.class.startsWith("Decimal")> holder.scale = getField().getScale(); holder.precision = getField().getPrecision(); </#if> - holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); + holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH); } public void get(int index, Nullable${minor.class}Holder holder){ holder.isSet = 1; - holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); + holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH); } </#if> <#-- type.width --> - } - - /** - * ${minor.class}.Mutator implements a mutable vector of fixed width values. Elements in the - * vector are accessed by position from the logical start of the vector. Values should be pushed - * onto the vector sequentially, but may be randomly accessed. - * The width of each element is ${type.width} byte(s) - * The equivalent Java primitive is '${minor.javaType!type.javaType}' - * - * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker. - */ - public final class Mutator extends BaseDataValueVector.BaseMutator { - - private Mutator(){}; - /** - * Set the element at the given index to the given value. Note that widths smaller than - * 32 bits are handled by the DrillBuf interface. - * - * @param index position of the bit to set - * @param value value to set - */ + } + + /** + * ${minor.class}.Mutator implements a mutable vector of fixed width values. Elements in the + * vector are accessed by position from the logical start of the vector. Values should be pushed + * onto the vector sequentially, but may be randomly accessed. + * <ul> + * <li>The width of each element is {@link #VALUE_WIDTH} (= ${type.width}) byte(s).</li> + * <li>The equivalent Java primitive is '${minor.javaType!type.javaType}'</li> + * </ul> + * + * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker. + */ + public final class Mutator extends BaseDataValueVector.BaseMutator { + + private Mutator() {}; + + /** + * Set the element at the given index to the given value. Note that widths smaller than + * 32 bits are handled by the DrillBuf interface. + * + * @param index position of the bit to set + * @param value value to set + */ + <#if (type.width > 8)> public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) { - data.setBytes(index * ${type.width}, value, 0, ${type.width}); + data.setBytes(index * VALUE_WIDTH, value, 0, VALUE_WIDTH); } public void setSafe(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) { while(index >= getValueCapacity()) { reAlloc(); } - data.setBytes(index * ${type.width}, value, 0, ${type.width}); + data.setBytes(index * VALUE_WIDTH, value, 0, VALUE_WIDTH); } - <#if (minor.class == "Interval")> - public void set(int index, int months, int days, int milliseconds){ - final int offsetIndex = index * ${type.width}; - data.setInt(offsetIndex, months); - data.setInt((offsetIndex + ${minor.daysOffset}), days); - data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds); + /** + * Set the value of a required or nullable vector. Enforces the value + * and size limits. + * @param index item to write + * @param value value to set + * @throws VectorOverflowException if the item was written, false if the index would + * overfill the vector + */ + + public void setScalar(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) throws VectorOverflowException { + if (index >= MAX_SCALAR_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, value); } - protected void set(int index, ${minor.class}Holder holder){ - set(index, holder.months, holder.days, holder.milliseconds); + /** + * Set the value of a repeated vector. Enforces only the size limit. + * @param index item to write + * @param value value to set + * @throws VectorOverflowException if the item was written, false if the index would + * overfill the vector + */ + + public void setArrayItem(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) throws VectorOverflowException { + if (index >= MAX_VALUE_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, value); } - protected void set(int index, Nullable${minor.class}Holder holder){ - set(index, holder.months, holder.days, holder.milliseconds); + <#if minor.class == "Interval"> + public void set(int index, int months, int days, int milliseconds) { + final int offsetIndex = index * VALUE_WIDTH; + data.setInt(offsetIndex, months); + data.setInt((offsetIndex + ${minor.daysOffset}), days); + data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds); } - public void setSafe(int index, int months, int days, int milliseconds){ + public void setSafe(int index, int months, int days, int milliseconds) { while(index >= getValueCapacity()) { reAlloc(); } set(index, months, days, milliseconds); } - public void setSafe(int index, Nullable${minor.class}Holder holder){ + public void setScalar(int index, int months, int days, int milliseconds) throws VectorOverflowException { + if (index >= MAX_SCALAR_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, months, days, milliseconds); + } + + public void setArrayItem(int index, int months, int days, int milliseconds) throws VectorOverflowException { + if (index >= MAX_VALUE_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, months, days, milliseconds); + } + + protected void set(int index, ${minor.class}Holder holder) { + set(index, holder.months, holder.days, holder.milliseconds); + } + + public void setSafe(int index, ${minor.class}Holder holder) { setSafe(index, holder.months, holder.days, holder.milliseconds); } - public void setSafe(int index, ${minor.class}Holder holder){ + public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException { + setScalar(index, holder.months, holder.days, holder.milliseconds); + } + + public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException { + setArrayItem(index, holder.months, holder.days, holder.milliseconds); + } + + protected void set(int index, Nullable${minor.class}Holder holder) { + set(index, holder.months, holder.days, holder.milliseconds); + } + + public void setSafe(int index, Nullable${minor.class}Holder holder) { setSafe(index, holder.months, holder.days, holder.milliseconds); } - <#elseif (minor.class == "IntervalDay")> - public void set(int index, int days, int milliseconds){ - final int offsetIndex = index * ${type.width}; - data.setInt(offsetIndex, days); - data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds); + public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { + setScalar(index, holder.months, holder.days, holder.milliseconds); } - protected void set(int index, ${minor.class}Holder holder){ - set(index, holder.days, holder.milliseconds); + public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { + setArrayItem(index, holder.months, holder.days, holder.milliseconds); } - protected void set(int index, Nullable${minor.class}Holder holder){ - set(index, holder.days, holder.milliseconds); + <#elseif minor.class == "IntervalDay"> + public void set(int index, int days, int milliseconds) { + final int offsetIndex = index * VALUE_WIDTH; + data.setInt(offsetIndex, days); + data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds); } - public void setSafe(int index, int days, int milliseconds){ + public void setSafe(int index, int days, int milliseconds) { while(index >= getValueCapacity()) { reAlloc(); } set(index, days, milliseconds); } - public void setSafe(int index, ${minor.class}Holder holder){ - setSafe(index, holder.days, holder.milliseconds); + public void setScalar(int index, int days, int milliseconds) throws VectorOverflowException { + if (index >= MAX_SCALAR_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, days, milliseconds); } - public void setSafe(int index, Nullable${minor.class}Holder holder){ + public void setArrayItem(int index, int days, int milliseconds) throws VectorOverflowException { + if (index >= MAX_VALUE_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, days, milliseconds); + } + + protected void set(int index, ${minor.class}Holder holder) { + set(index, holder.days, holder.milliseconds); + } + + public void setSafe(int index, ${minor.class}Holder holder) { setSafe(index, holder.days, holder.milliseconds); } - <#elseif (minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")> - public void set(int index, ${minor.class}Holder holder){ - set(index, holder.start, holder.buffer); + public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException { + setScalar(index, holder.days, holder.milliseconds); } - void set(int index, Nullable${minor.class}Holder holder){ - set(index, holder.start, holder.buffer); + public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException { + setArrayItem(index, holder.days, holder.milliseconds); } - public void setSafe(int index, Nullable${minor.class}Holder holder){ - setSafe(index, holder.start, holder.buffer); + protected void set(int index, Nullable${minor.class}Holder holder) { + set(index, holder.days, holder.milliseconds); } - public void setSafe(int index, ${minor.class}Holder holder){ - setSafe(index, holder.start, holder.buffer); + public void setSafe(int index, Nullable${minor.class}Holder holder){ + setSafe(index, holder.days, holder.milliseconds); + } + + public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { + setScalar(index, holder.days, holder.milliseconds); } - public void setSafe(int index, int start, DrillBuf buffer){ + public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { + setArrayItem(index, holder.days, holder.milliseconds); + } + + <#elseif minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense"> + public void setSafe(int index, int start, DrillBuf buffer) { while(index >= getValueCapacity()) { reAlloc(); } set(index, start, buffer); } - <#if minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse"> - public void set(int index, BigDecimal value) { - DecimalUtility.getSparseFromBigDecimal(value, data, index * ${type.width}, - field.getScale(), field.getPrecision(), ${minor.nDecimalDigits}); + public void setScalar(int index, int start, DrillBuf buffer) throws VectorOverflowException { + if (index >= MAX_SCALAR_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, start, buffer); } - public void setSafe(int index, BigDecimal value) { - while(index >= getValueCapacity()) { - reAlloc(); + public void setArrayItem(int index, int start, DrillBuf buffer) throws VectorOverflowException { + if (index >= MAX_VALUE_COUNT) { + throw new VectorOverflowException(); } - set(index, value); + setSafe(index, start, buffer); } - </#if> - public void set(int index, int start, DrillBuf buffer){ - data.setBytes(index * ${type.width}, buffer, start, ${type.width}); + public void set(int index, ${minor.class}Holder holder) { + set(index, holder.start, holder.buffer); } - <#else> - protected void set(int index, ${minor.class}Holder holder){ - set(index, holder.start, holder.buffer); + public void setSafe(int index, ${minor.class}Holder holder) { + setSafe(index, holder.start, holder.buffer); } - public void set(int index, Nullable${minor.class}Holder holder){ - set(index, holder.start, holder.buffer); + public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException { + setScalar(index, holder.start, holder.buffer); } - public void set(int index, int start, DrillBuf buffer){ - data.setBytes(index * ${type.width}, buffer, start, ${type.width}); + public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException { + setArrayItem(index, holder.start, holder.buffer); } - public void setSafe(int index, ${minor.class}Holder holder){ - setSafe(index, holder.start, holder.buffer); + void set(int index, Nullable${minor.class}Holder holder) { + set(index, holder.start, holder.buffer); } - public void setSafe(int index, Nullable${minor.class}Holder holder){ + public void setSafe(int index, Nullable${minor.class}Holder holder) { setSafe(index, holder.start, holder.buffer); } - public void setSafe(int index, int start, DrillBuf buffer){ + public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { + setScalar(index, holder.start, holder.buffer); + } + + public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { + setArrayItem(index, holder.start, holder.buffer); + } + + <#if minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse"> + public void set(int index, BigDecimal value) { + DecimalUtility.getSparseFromBigDecimal(value, data, index * VALUE_WIDTH, + field.getScale(), field.getPrecision(), ${minor.nDecimalDigits}); + } + + public void setSafe(int index, BigDecimal value) { while(index >= getValueCapacity()) { reAlloc(); } - set(index, holder); + set(index, value); } - public void set(int index, Nullable${minor.class}Holder holder){ - data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width}); + public void setScalar(int index, BigDecimal value) throws VectorOverflowException { + if (index >= MAX_SCALAR_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, value); } - </#if> + public void setArrayItem(int index, BigDecimal value) throws VectorOverflowException { + if (index >= MAX_VALUE_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, value); + } + + </#if> + public void set(int index, int start, DrillBuf buffer){ + data.setBytes(index * VALUE_WIDTH, buffer, start, VALUE_WIDTH); + } + + </#if> @Override public void generateTestData(int count) { setValueCount(count); @@ -738,46 +879,105 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F final int valueCount = getAccessor().getValueCount(); for(int i = 0; i < valueCount; i++, even = !even) { final byte b = even ? Byte.MIN_VALUE : Byte.MAX_VALUE; - for(int w = 0; w < ${type.width}; w++){ + for(int w = 0; w < VALUE_WIDTH; w++){ data.setByte(i + w, b); } } } - <#else> <#-- type.width <= 8 --> + <#else> <#-- type.width <= 8 --> public void set(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) { - data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, value); + data.set${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH, value); } - public void setSafe(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) { - while(index >= getValueCapacity()) { + public void setSafe(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) { + while(index >= getValueCapacity()) { reAlloc(); } set(index, value); } - protected void set(int index, ${minor.class}Holder holder){ - data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value); + /** + * Set the value of a required or nullable vector. Enforces the value + * and size limits. + * @param index item to write + * @param value value to set + * @throws VectorOverflowException if the item was written, false if the index would + * overfill the vector + */ + + public void setScalar(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) throws VectorOverflowException { + if (index >= MAX_SCALAR_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, value); + } + + /** + * Set the value of a repeated vector. Enforces only the size limit. + * @param index item to write + * @param value value to set + * @throws VectorOverflowException if the item was written, false if the index would + * overfill the vector + */ + + public void setArrayItem(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) throws VectorOverflowException { + if (index >= MAX_VALUE_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, value); + } + + protected void set(int index, ${minor.class}Holder holder) { + data.set${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH, holder.value); } - public void setSafe(int index, ${minor.class}Holder holder){ + public void setSafe(int index, ${minor.class}Holder holder) { while(index >= getValueCapacity()) { reAlloc(); } set(index, holder); } - protected void set(int index, Nullable${minor.class}Holder holder){ - data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value); + public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException { + if (index >= MAX_SCALAR_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, holder); } - public void setSafe(int index, Nullable${minor.class}Holder holder){ + public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException { + if (index >= MAX_VALUE_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, holder); + } + + protected void set(int index, Nullable${minor.class}Holder holder) { + data.set${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH, holder.value); + } + + public void setSafe(int index, Nullable${minor.class}Holder holder) { while(index >= getValueCapacity()) { reAlloc(); } set(index, holder); } + public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { + if (index >= MAX_SCALAR_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, holder); + } + + public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { + if (index >= MAX_VALUE_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, holder); + } + @Override public void generateTestData(int size) { setValueCount(size); @@ -806,10 +1006,34 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } </#if> <#-- type.width --> + /** + * Backfill missing offsets from the given last written position to the + * given current write position. Used by the "new" size-safe column + * writers to allow skipping values. The <tt>set()</tt> and <tt>setSafe()</tt> + * <b>do not</b> fill empties. See DRILL-5529 and DRILL-5530. + * @param lastWrite the position of the last valid write: the offset + * to be copied forward + * @param index the current write position filling occurs up to, + * but not including, this position + * @throws VectorOverflowException if the item was written, false if the index would + * overfill the vector + */ + + public void fillEmptiesBounded(int lastWrite, int index) + throws VectorOverflowException { + <#if type.width <= 8> + for (int i = lastWrite + 1; i <= index; i++) { + setSafe(i, <#if (type.width >= 4)>(${minor.javaType!type.javaType})</#if> 0); + } + <#else> + throw new UnsupportedOperationException("Cannot zero-fill ${minor.class} vectors."); + </#if> + } + @Override public void setValueCount(int valueCount) { final int currentValueCapacity = getValueCapacity(); - final int idx = (${type.width} * valueCount); + final int idx = (VALUE_WIDTH * valueCount); while(valueCount > getValueCapacity()) { reAlloc(); } @@ -819,11 +1043,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F allocationMonitor = 0; } VectorTrimmer.trim(data, idx); - data.writerIndex(valueCount * ${type.width}); + data.writerIndex(valueCount * VALUE_WIDTH); } } } - </#if> <#-- type.major --> </#list> </#list>
