Repository: drill Updated Branches: refs/heads/master b4d47c56b -> 27364122c
DRILL-3920: Additional tests added to TestValueVectors for serialization and loading. Some light cleanup of a few vector implementations. closes #194 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/27364122 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/27364122 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/27364122 Branch: refs/heads/master Commit: 27364122ca0a82c6dd5552f292b2711bb2b28cf5 Parents: b4d47c5 Author: Chris Westin <cwes...@yahoo.com> Authored: Thu Oct 8 18:15:57 2015 -0700 Committer: Hanifi Gunes <hgu...@maprtech.com> Committed: Tue Oct 13 16:35:22 2015 -0700 ---------------------------------------------------------------------- .../drill/exec/vector/BaseValueVector.java | 16 +- .../apache/drill/exec/vector/ValueVector.java | 12 +- .../exec/vector/complex/AbstractMapVector.java | 55 +-- .../drill/exec/vector/complex/MapVector.java | 24 +- .../exec/record/vector/TestValueVector.java | 352 ++++++++++++++----- 5 files changed, 329 insertions(+), 130 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/27364122/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java index cc287c4..7b3ab41 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java @@ -17,10 +17,13 @@ */ package org.apache.drill.exec.vector; +import io.netty.buffer.DrillBuf; + import java.util.Iterator; import com.google.common.base.Preconditions; import com.google.common.collect.Iterators; + import org.apache.drill.common.expression.FieldReference; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.proto.UserBitShared.SerializedField; @@ -91,10 +94,10 @@ public abstract class BaseValueVector implements ValueVector { protected BaseMutator() { } @Override - public void generateTestData(int values) { } + public void generateTestData(int values) {} //TODO: consider making mutator stateless(if possible) on another issue. - public void reset() { } + public void reset() {} } @Override @@ -102,5 +105,14 @@ public abstract class BaseValueVector implements ValueVector { return Iterators.emptyIterator(); } + public static boolean checkBufRefs(final ValueVector vv) { + for(final DrillBuf buffer : vv.getBuffers(false)) { + if (buffer.refCnt() <= 0) { + throw new IllegalStateException("zero refcount"); + } + } + + return true; + } } http://git-wip-us.apache.org/repos/asf/drill/blob/27364122/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java index 691d1b7..8627d1a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java @@ -17,10 +17,10 @@ */ package org.apache.drill.exec.vector; -import io.netty.buffer.DrillBuf; - import java.io.Closeable; +import io.netty.buffer.DrillBuf; + import org.apache.drill.common.expression.FieldReference; import org.apache.drill.exec.memory.OutOfMemoryRuntimeException; import org.apache.drill.exec.proto.UserBitShared.SerializedField; @@ -80,8 +80,9 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> { int getValueCapacity(); /** - * Alternative to clear(). Allows use as closeable in try-with-resources. + * Alternative to clear(). Allows use as an AutoCloseable in try-with-resources. */ + @Override void close(); /** @@ -155,9 +156,8 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> { * Return the underlying buffers associated with this vector. Note that this doesn't impact the reference counts for * this buffer so it only should be used for in-context access. Also note that this buffer changes regularly thus * external classes shouldn't hold a reference to it (unless they change it). - * - * @param clear - * Whether to clear vector + * @param clear Whether to clear vector before returning; the buffers will still be refcounted; + * but the returned array will be the only reference to them * * @return The underlying {@link io.netty.buffer.DrillBuf buffers} that is used by this vector instance. */ http://git-wip-us.apache.org/repos/asf/drill/blob/27364122/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java index efba46d..2c93c31 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java @@ -36,6 +36,7 @@ import java.util.List; * Base class for MapVectors. Currently used by RepeatedMapVector and MapVector */ public abstract class AbstractMapVector extends AbstractContainerVector { + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class); // Maintains a map with key as field name and value is the vector itself private final MapWithOrdinal<String, ValueVector> vectors = new MapWithOrdinal<>(); @@ -46,14 +47,24 @@ public abstract class AbstractMapVector extends AbstractContainerVector { // create the hierarchy of the child vectors based on the materialized field for (MaterializedField child : clonedField.getChildren()) { if (!child.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) { - String fieldName = child.getLastName(); - ValueVector v = TypeHelper.getNewVector(child, allocator, callBack); + final String fieldName = child.getLastName(); + final ValueVector v = TypeHelper.getNewVector(child, allocator, callBack); putVector(fieldName, v); } } } @Override + public void close() { + for(final ValueVector valueVector : vectors.values()) { + valueVector.close(); + } + vectors.clear(); + + super.close(); + } + + @Override public boolean allocateNewSafe() { /* boolean to keep track if all the memory allocation were successful * Used in the case of composite vectors when we need to allocate multiple @@ -62,8 +73,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector { */ boolean success = false; try { - - for (ValueVector v : vectors.values()) { + for (final ValueVector v : vectors.values()) { if (!v.allocateNewSafe()) { return false; } @@ -105,13 +115,14 @@ public abstract class AbstractMapVector extends AbstractContainerVector { * * @return resultant {@link org.apache.drill.exec.vector.ValueVector} */ + @Override public <T extends ValueVector> T addOrGet(String name, TypeProtos.MajorType type, Class<T> clazz) { final ValueVector existing = getChild(name); boolean create = false; if (existing == null) { create = true; } else if (clazz.isAssignableFrom(existing.getClass())) { - return (T)existing; + return (T) existing; } else if (nullFilled(existing)) { existing.clear(); create = true; @@ -129,7 +140,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector { } private boolean nullFilled(ValueVector vector) { - for (int r=0; r<vector.getAccessor().getValueCount(); r++) { + for (int r = 0; r < vector.getAccessor().getValueCount(); r++) { if (!vector.getAccessor().isNull(r)) { return false; } @@ -148,8 +159,9 @@ public abstract class AbstractMapVector extends AbstractContainerVector { * Returns a {@link org.apache.drill.exec.vector.ValueVector} instance of subtype of <T> corresponding to the given * field name if exists or null. */ + @Override public <T extends ValueVector> T getChild(String name, Class<T> clazz) { - ValueVector v = vectors.get(name.toLowerCase()); + final ValueVector v = vectors.get(name.toLowerCase()); if (v == null) { return null; } @@ -172,7 +184,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector { * @param vector vector to be inserted */ protected void putVector(String name, ValueVector vector) { - ValueVector old = vectors.put( + final ValueVector old = vectors.put( Preconditions.checkNotNull(name, "field name cannot be null").toLowerCase(), Preconditions.checkNotNull(vector, "vector cannot be null") ); @@ -192,6 +204,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector { /** * Returns the number of underlying child vectors. */ + @Override public int size() { return vectors.size(); } @@ -205,8 +218,8 @@ public abstract class AbstractMapVector extends AbstractContainerVector { * Returns a list of scalar child vectors recursing the entire vector hierarchy. */ public List<ValueVector> getPrimitiveVectors() { - List<ValueVector> primitiveVectors = Lists.newArrayList(); - for (ValueVector v : vectors.values()) { + final List<ValueVector> primitiveVectors = Lists.newArrayList(); + for (final ValueVector v : vectors.values()) { if (v instanceof AbstractMapVector) { AbstractMapVector mapVector = (AbstractMapVector) v; primitiveVectors.addAll(mapVector.getPrimitiveVectors()); @@ -220,6 +233,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector { /** * Returns a vector with its corresponding ordinal mapping if field exists or null. */ + @Override public VectorWithOrdinal getChildVectorWithOrdinal(String name) { final int ordinal = vectors.getOrdinal(name.toLowerCase()); if (ordinal < 0) { @@ -231,13 +245,13 @@ public abstract class AbstractMapVector extends AbstractContainerVector { @Override public DrillBuf[] getBuffers(boolean clear) { - List<DrillBuf> buffers = Lists.newArrayList(); + final List<DrillBuf> buffers = Lists.newArrayList(); - for (ValueVector vector : vectors.values()) { - for (DrillBuf buf : vector.getBuffers(false)) { + for (final ValueVector vector : vectors.values()) { + for (final DrillBuf buf : vector.getBuffers(false)) { buffers.add(buf); if (clear) { - buf.retain(); + buf.retain(1); } } if (clear) { @@ -252,20 +266,11 @@ public abstract class AbstractMapVector extends AbstractContainerVector { public int getBufferSize() { int actualBufSize = 0 ; - for (ValueVector v : vectors.values()) { - for (DrillBuf buf : v.getBuffers(false)) { + for (final ValueVector v : vectors.values()) { + for (final DrillBuf buf : v.getBuffers(false)) { actualBufSize += buf.writerIndex(); } } return actualBufSize; } - - @Override - public void close() { - for(final ValueVector valueVector : vectors.values()) { - valueVector.close(); - } - - super.close(); - } } http://git-wip-us.apache.org/repos/asf/drill/blob/27364122/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java index 66f7962..8b4b858 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java @@ -22,6 +22,7 @@ import com.google.common.primitives.Ints; import io.netty.buffer.DrillBuf; +import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -50,7 +51,7 @@ import org.apache.drill.exec.vector.complex.reader.FieldReader; import com.google.common.base.Preconditions; public class MapVector extends AbstractMapVector { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class); + //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class); public final static MajorType TYPE = Types.required(MinorType.MAP); @@ -112,7 +113,7 @@ public class MapVector extends AbstractMapVector { return 0; } long buffer = 0; - for (ValueVector v : (Iterable<ValueVector>)this) { + for (final ValueVector v : (Iterable<ValueVector>)this) { buffer += v.getBufferSize(); } @@ -149,7 +150,7 @@ public class MapVector extends AbstractMapVector { @Override public TransferPair makeTransferPair(ValueVector to) { - return new MapTransferPair(this, (MapVector)to); + return new MapTransferPair(this, (MapVector) to); } @Override @@ -194,7 +195,7 @@ public class MapVector extends AbstractMapVector { // (This is similar to what happens in ScanBatch where the children cannot be added till they are // read). To take care of this, we ensure that the hashCode of the MaterializedField does not // include the hashCode of the children but is based only on MaterializedField$key. - ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass()); + final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass()); if (allocate && to.size() != preSize) { newVector.allocateNew(); } @@ -202,10 +203,9 @@ public class MapVector extends AbstractMapVector { } } - @Override public void transfer() { - for (TransferPair p : pairs) { + for (final TransferPair p : pairs) { p.transfer(); } to.valueCount = from.valueCount; @@ -231,7 +231,6 @@ public class MapVector extends AbstractMapVector { } to.getMutator().setValueCount(length); } - } @Override @@ -329,7 +328,6 @@ public class MapVector extends AbstractMapVector { public int getValueCount() { return valueCount; } - } public ValueVector getVectorById(int id) { @@ -340,7 +338,7 @@ public class MapVector extends AbstractMapVector { @Override public void setValueCount(int valueCount) { - for (ValueVector v : getChildren()) { + for (final ValueVector v : getChildren()) { v.getMutator().setValueCount(valueCount); } MapVector.this.valueCount = valueCount; @@ -355,17 +353,19 @@ public class MapVector extends AbstractMapVector { @Override public void clear() { - valueCount = 0; - for (ValueVector v : getChildren()) { + for (final ValueVector v : getChildren()) { v.clear(); } + valueCount = 0; } @Override public void close() { - for (final ValueVector v : getChildren()) { + final Collection<ValueVector> vectors = getChildren(); + for (final ValueVector v : vectors) { v.close(); } + vectors.clear(); valueCount = 0; super.close(); http://git-wip-us.apache.org/repos/asf/drill/blob/27364122/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java index af4f2f0..0a9b470 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java @@ -20,12 +20,13 @@ package org.apache.drill.exec.record.vector; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import io.netty.buffer.DrillBuf; import java.nio.charset.Charset; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; -import io.netty.buffer.DrillBuf; + import org.apache.drill.common.AutoCloseables; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.SchemaPath; @@ -41,6 +42,7 @@ import org.apache.drill.exec.expr.holders.NullableUInt4Holder; import org.apache.drill.exec.expr.holders.NullableVar16CharHolder; import org.apache.drill.exec.expr.holders.NullableVarCharHolder; import org.apache.drill.exec.expr.holders.RepeatedFloat4Holder; +import org.apache.drill.exec.expr.holders.RepeatedIntHolder; import org.apache.drill.exec.expr.holders.RepeatedVarBinaryHolder; import org.apache.drill.exec.expr.holders.UInt1Holder; import org.apache.drill.exec.expr.holders.UInt4Holder; @@ -54,6 +56,7 @@ import org.apache.drill.exec.vector.BitVector; import org.apache.drill.exec.vector.NullableFloat4Vector; import org.apache.drill.exec.vector.NullableUInt4Vector; import org.apache.drill.exec.vector.NullableVarCharVector; +import org.apache.drill.exec.vector.RepeatedIntVector; import org.apache.drill.exec.vector.UInt4Vector; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.VarCharVector; @@ -65,11 +68,9 @@ import org.junit.Before; import org.junit.Test; public class TestValueVector extends ExecTest { - private final static SchemaPath EMPTY_SCHEMA_PATH = SchemaPath.getSimplePath(""); + //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestValueVector.class); - private final static byte[] STR1 = new String("AAAAA1").getBytes(Charset.forName("UTF-8")); - private final static byte[] STR2 = new String("BBBBBBBBB2").getBytes(Charset.forName("UTF-8")); - private final static byte[] STR3 = new String("CCCC3").getBytes(Charset.forName("UTF-8")); + private final static SchemaPath EMPTY_SCHEMA_PATH = SchemaPath.getSimplePath(""); private DrillConfig drillConfig; private BufferAllocator allocator; @@ -80,12 +81,16 @@ public class TestValueVector extends ExecTest { allocator = RootAllocatorFactory.newRoot(drillConfig); } + private final static Charset utf8Charset = Charset.forName("UTF-8"); + private final static byte[] STR1 = new String("AAAAA1").getBytes(utf8Charset); + private final static byte[] STR2 = new String("BBBBBBBBB2").getBytes(utf8Charset); + private final static byte[] STR3 = new String("CCCC3").getBytes(utf8Charset); + @After - public void terminate() { + public void terminate() throws Exception { allocator.close(); } - @Test(expected = OversizedAllocationException.class) public void testFixedVectorReallocation() { final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE); @@ -172,11 +177,11 @@ public class TestValueVector extends ExecTest { @Test public void testFixedType() { - MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE); + final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE); - // Create a new value vector for 1024 integers - try (UInt4Vector vector = new UInt4Vector(field, allocator)) { - UInt4Vector.Mutator m = vector.getMutator(); + // Create a new value vector for 1024 integers. + try (final UInt4Vector vector = new UInt4Vector(field, allocator)) { + final UInt4Vector.Mutator m = vector.getMutator(); vector.allocateNew(1024); // Put and set a few values @@ -185,33 +190,36 @@ public class TestValueVector extends ExecTest { m.setSafe(100, 102); m.setSafe(1022, 103); m.setSafe(1023, 104); - assertEquals(100, vector.getAccessor().get(0)); - assertEquals(101, vector.getAccessor().get(1)); - assertEquals(102, vector.getAccessor().get(100)); - assertEquals(103, vector.getAccessor().get(1022)); - assertEquals(104, vector.getAccessor().get(1023)); + + final UInt4Vector.Accessor accessor = vector.getAccessor(); + assertEquals(100, accessor.get(0)); + assertEquals(101, accessor.get(1)); + assertEquals(102, accessor.get(100)); + assertEquals(103, accessor.get(1022)); + assertEquals(104, accessor.get(1023)); } } @Test public void testNullableVarLen2() { - MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE); + final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE); - // Create a new value vector for 1024 integers - try (NullableVarCharVector vector = new NullableVarCharVector(field, allocator)) { - NullableVarCharVector.Mutator m = vector.getMutator(); + // Create a new value vector for 1024 integers. + try (final NullableVarCharVector vector = new NullableVarCharVector(field, allocator)) { + final NullableVarCharVector.Mutator m = vector.getMutator(); vector.allocateNew(1024 * 10, 1024); m.set(0, STR1); m.set(1, STR2); m.set(2, STR3); - // Check the sample strings - assertArrayEquals(STR1, vector.getAccessor().get(0)); - assertArrayEquals(STR2, vector.getAccessor().get(1)); - assertArrayEquals(STR3, vector.getAccessor().get(2)); + // Check the sample strings. + final NullableVarCharVector.Accessor accessor = vector.getAccessor(); + assertArrayEquals(STR1, accessor.get(0)); + assertArrayEquals(STR2, accessor.get(1)); + assertArrayEquals(STR3, accessor.get(2)); - // Ensure null value throws + // Ensure null value throws. boolean b = false; try { vector.getAccessor().get(3); @@ -223,14 +231,185 @@ public class TestValueVector extends ExecTest { } } + private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[] buffers) { + // find the total size we'll need + int size = 0; + for(final DrillBuf buffer : buffers) { + size += buffer.readableBytes(); + } + + // create the new buffer + final DrillBuf newBuf = allocator.buffer(size); + final DrillBuf writeBuf = newBuf; + for(final DrillBuf buffer : buffers) { + final DrillBuf readBuf = (DrillBuf) buffer.slice(); + final int nBytes = readBuf.readableBytes(); + final byte[] bytes = new byte[nBytes]; + readBuf.readBytes(bytes); + writeBuf.writeBytes(bytes); + } + + return newBuf; + } + + @Test + public void testRepeatedIntVector() { + final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE); + + // Create a new value vector. + final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator); + + // Populate the vector. + final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes + final int nRecords = 7; + final int nElements = values.length; + vector1.allocateNew(nRecords, nRecords * nElements); + final RepeatedIntVector.Mutator mutator = vector1.getMutator(); + for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) { + mutator.startNewValue(recordIndex); + for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) { + mutator.add(recordIndex, recordIndex * values[elementIndex]); + } + } + mutator.setValueCount(nRecords); + + // Verify the contents. + final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor(); + assertEquals(nRecords, accessor1.getValueCount()); + for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) { + for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) { + final int value = accessor1.get(recordIndex, elementIndex); + assertEquals(recordIndex * values[elementIndex], value); + } + } + +/* TODO(cwestin) +the interface to load has changed + // Serialize, reify, and verify. + final DrillBuf[] buffers1 = vector1.getBuffers(false); + final DrillBuf buffer1 = combineBuffers(allocator, buffers1); + final RepeatedIntVector vector2 = new RepeatedIntVector(field, allocator); + vector2.load(nRecords, nRecords * nElements, buffer1); + + final RepeatedIntVector.Accessor accessor2 = vector2.getAccessor(); + for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) { + for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) { + final int value = accessor2.get(recordIndex, elementIndex); + assertEquals(accessor1.get(recordIndex, elementIndex), value); + } + } +*/ + + vector1.close(); +/* TODO(cwestin) + vector2.close(); + buffer1.release(); +*/ + } + + @Test + public void testVarCharVectorLoad() { + final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE); + + // Create a new value vector for 1024 variable length strings. + final VarCharVector vector1 = new VarCharVector(field, allocator); + final VarCharVector.Mutator mutator = vector1.getMutator(); + vector1.allocateNew(1024 * 10, 1024); + + // Populate the vector. + final StringBuilder stringBuilder = new StringBuilder(); + final int valueCount = 10; + for(int i = 0; i < valueCount; ++i) { + stringBuilder.append('x'); + mutator.setSafe(i, stringBuilder.toString().getBytes(utf8Charset)); + } + mutator.setValueCount(valueCount); + assertEquals(valueCount, vector1.getAccessor().getValueCount()); + + // Combine the backing buffers so we can load them into a new vector. + final DrillBuf[] buffers1 = vector1.getBuffers(false); + final DrillBuf buffer1 = combineBuffers(allocator, buffers1); + final VarCharVector vector2 = new VarCharVector(field, allocator); + vector2.load(vector1.getMetadata(), buffer1); + + // Check the contents of the new vector. + final VarCharVector.Accessor accessor = vector2.getAccessor(); + stringBuilder.setLength(0); + for(int i = 0; i < valueCount; ++i) { + stringBuilder.append('x'); + final Object object = accessor.getObject(i); + assertEquals(stringBuilder.toString(), object.toString()); + } + + vector1.close(); + vector2.close(); + buffer1.release(); + } + + @Test + public void testNullableVarCharVectorLoad() { + final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE); + + // Create a new value vector for 1024 nullable variable length strings. + final NullableVarCharVector vector1 = new NullableVarCharVector(field, allocator); + final NullableVarCharVector.Mutator mutator = vector1.getMutator(); + vector1.allocateNew(1024 * 10, 1024); + + // Populate the vector. + final StringBuilder stringBuilder = new StringBuilder(); + final int valueCount = 10; + for(int i = 0; i < valueCount; ++i) { + stringBuilder.append('x'); + mutator.set(i, stringBuilder.toString().getBytes(utf8Charset)); + } + + // Check the contents. + final NullableVarCharVector.Accessor accessor1 = vector1.getAccessor(); + stringBuilder.setLength(0); + for(int i = 0; i < valueCount; ++i) { + stringBuilder.append('x'); + final Object object = accessor1.getObject(i); + assertEquals(stringBuilder.toString(), object.toString()); + } + + mutator.setValueCount(valueCount); + assertEquals(valueCount, vector1.getAccessor().getValueCount()); + + // Still ok after setting value count? + stringBuilder.setLength(0); + for(int i = 0; i < valueCount; ++i) { + stringBuilder.append('x'); + final Object object = accessor1.getObject(i); + assertEquals(stringBuilder.toString(), object.toString()); + } + + // Combine into a single buffer so we can load it into a new vector. + final DrillBuf[] buffers1 = vector1.getBuffers(false); + final DrillBuf buffer1 = combineBuffers(allocator, buffers1); + final NullableVarCharVector vector2 = new NullableVarCharVector(field, allocator); + vector2.load(vector1.getMetadata(), buffer1); + + // Check the vector's contents. + final NullableVarCharVector.Accessor accessor2 = vector2.getAccessor(); + stringBuilder.setLength(0); + for(int i = 0; i < valueCount; ++i) { + stringBuilder.append('x'); + final Object object = accessor2.getObject(i); + assertEquals(stringBuilder.toString(), object.toString()); + } + + vector1.close(); + vector2.close(); + buffer1.release(); + } @Test public void testNullableFixedType() { - MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableUInt4Holder.TYPE); + final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableUInt4Holder.TYPE); - // Create a new value vector for 1024 integers - try (NullableUInt4Vector vector = new NullableUInt4Vector(field, allocator)) { - NullableUInt4Vector.Mutator m = vector.getMutator(); + // Create a new value vector for 1024 integers. + try (final NullableUInt4Vector vector = new NullableUInt4Vector(field, allocator)) { + final NullableUInt4Vector.Mutator m = vector.getMutator(); vector.allocateNew(1024); // Put and set a few values @@ -239,17 +418,19 @@ public class TestValueVector extends ExecTest { m.set(100, 102); m.set(1022, 103); m.set(1023, 104); - assertEquals(100, vector.getAccessor().get(0)); - assertEquals(101, vector.getAccessor().get(1)); - assertEquals(102, vector.getAccessor().get(100)); - assertEquals(103, vector.getAccessor().get(1022)); - assertEquals(104, vector.getAccessor().get(1023)); + + final NullableUInt4Vector.Accessor accessor = vector.getAccessor(); + assertEquals(100, accessor.get(0)); + assertEquals(101, accessor.get(1)); + assertEquals(102, accessor.get(100)); + assertEquals(103, accessor.get(1022)); + assertEquals(104, accessor.get(1023)); // Ensure null values throw { boolean b = false; try { - vector.getAccessor().get(3); + accessor.get(3); } catch (IllegalStateException e) { b = true; } finally { @@ -257,12 +438,11 @@ public class TestValueVector extends ExecTest { } } - vector.allocateNew(2048); { boolean b = false; try { - vector.getAccessor().get(0); + accessor.get(0); } catch (IllegalStateException e) { b = true; } finally { @@ -275,14 +455,13 @@ public class TestValueVector extends ExecTest { m.set(100, 102); m.set(1022, 103); m.set(1023, 104); - assertEquals(100, vector.getAccessor().get(0)); - assertEquals(101, vector.getAccessor().get(1)); - assertEquals(102, vector.getAccessor().get(100)); - assertEquals(103, vector.getAccessor().get(1022)); - assertEquals(104, vector.getAccessor().get(1023)); - - // Ensure null values throw + assertEquals(100, accessor.get(0)); + assertEquals(101, accessor.get(1)); + assertEquals(102, accessor.get(100)); + assertEquals(103, accessor.get(1022)); + assertEquals(104, accessor.get(1023)); + // Ensure null values throw. { boolean b = false; try { @@ -294,31 +473,32 @@ public class TestValueVector extends ExecTest { } } } - } @Test public void testNullableFloat() { - MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableFloat4Holder.TYPE); + final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableFloat4Holder.TYPE); // Create a new value vector for 1024 integers - try (NullableFloat4Vector vector = (NullableFloat4Vector) TypeHelper.getNewVector(field, allocator)) { - NullableFloat4Vector.Mutator m = vector.getMutator(); + try (final NullableFloat4Vector vector = (NullableFloat4Vector) TypeHelper.getNewVector(field, allocator)) { + final NullableFloat4Vector.Mutator m = vector.getMutator(); vector.allocateNew(1024); - // Put and set a few values + // Put and set a few values. m.set(0, 100.1f); m.set(1, 101.2f); m.set(100, 102.3f); m.set(1022, 103.4f); m.set(1023, 104.5f); - assertEquals(100.1f, vector.getAccessor().get(0), 0); - assertEquals(101.2f, vector.getAccessor().get(1), 0); - assertEquals(102.3f, vector.getAccessor().get(100), 0); - assertEquals(103.4f, vector.getAccessor().get(1022), 0); - assertEquals(104.5f, vector.getAccessor().get(1023), 0); - // Ensure null values throw + final NullableFloat4Vector.Accessor accessor = vector.getAccessor(); + assertEquals(100.1f, accessor.get(0), 0); + assertEquals(101.2f, accessor.get(1), 0); + assertEquals(102.3f, accessor.get(100), 0); + assertEquals(103.4f, accessor.get(1022), 0); + assertEquals(104.5f, accessor.get(1023), 0); + + // Ensure null values throw. { boolean b = false; try { @@ -334,7 +514,7 @@ public class TestValueVector extends ExecTest { { boolean b = false; try { - vector.getAccessor().get(0); + accessor.get(0); } catch (IllegalStateException e) { b = true; } finally { @@ -346,11 +526,11 @@ public class TestValueVector extends ExecTest { @Test public void testBitVector() { - MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE); + final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE); // Create a new value vector for 1024 integers - try (BitVector vector = new BitVector(field, allocator)) { - BitVector.Mutator m = vector.getMutator(); + try (final BitVector vector = new BitVector(field, allocator)) { + final BitVector.Mutator m = vector.getMutator(); vector.allocateNew(1024); // Put and set a few values @@ -358,38 +538,39 @@ public class TestValueVector extends ExecTest { m.set(1, 0); m.set(100, 0); m.set(1022, 1); - assertEquals(1, vector.getAccessor().get(0)); - assertEquals(0, vector.getAccessor().get(1)); - assertEquals(0, vector.getAccessor().get(100)); - assertEquals(1, vector.getAccessor().get(1022)); + + final BitVector.Accessor accessor = vector.getAccessor(); + assertEquals(1, accessor.get(0)); + assertEquals(0, accessor.get(1)); + assertEquals(0, accessor.get(100)); + assertEquals(1, accessor.get(1022)); // test setting the same value twice m.set(0, 1); m.set(0, 1); m.set(1, 0); m.set(1, 0); - assertEquals(1, vector.getAccessor().get(0)); - assertEquals(0, vector.getAccessor().get(1)); + assertEquals(1, accessor.get(0)); + assertEquals(0, accessor.get(1)); // test toggling the values m.set(0, 0); m.set(1, 1); - assertEquals(0, vector.getAccessor().get(0)); - assertEquals(1, vector.getAccessor().get(1)); + assertEquals(0, accessor.get(0)); + assertEquals(1, accessor.get(1)); // Ensure unallocated space returns 0 - assertEquals(0, vector.getAccessor().get(3)); + assertEquals(0, accessor.get(3)); } } - @Test - public void testReAllocNullableFixedWidthVector() throws Exception { - MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableFloat4Holder.TYPE); + public void testReAllocNullableFixedWidthVector() { + final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableFloat4Holder.TYPE); // Create a new value vector for 1024 integers - try (NullableFloat4Vector vector = (NullableFloat4Vector) TypeHelper.getNewVector(field, allocator)) { - NullableFloat4Vector.Mutator m = vector.getMutator(); + try (final NullableFloat4Vector vector = (NullableFloat4Vector) TypeHelper.getNewVector(field, allocator)) { + final NullableFloat4Vector.Mutator m = vector.getMutator(); vector.allocateNew(1024); assertEquals(1024, vector.getValueCapacity()); @@ -405,11 +586,11 @@ public class TestValueVector extends ExecTest { // Check valueCapacity is more than initial allocation assertEquals(1024 * 2, vector.getValueCapacity()); - assertEquals(100.1f, vector.getAccessor().get(0), 0); - assertEquals(102.3f, vector.getAccessor().get(100), 0); - assertEquals(104.5f, vector.getAccessor().get(1023), 0); - assertEquals(105.5f, vector.getAccessor().get(2000), 0); - + final NullableFloat4Vector.Accessor accessor = vector.getAccessor(); + assertEquals(100.1f, accessor.get(0), 0); + assertEquals(102.3f, accessor.get(100), 0); + assertEquals(104.5f, accessor.get(1023), 0); + assertEquals(105.5f, accessor.get(2000), 0); // Set the valueCount to be more than valueCapacity of current allocation. This is possible for NullableValueVectors // as we don't call setSafe for null values, but we do call setValueCount when all values are inserted into the @@ -419,12 +600,12 @@ public class TestValueVector extends ExecTest { } @Test - public void testReAllocNullableVariableWidthVector() throws Exception { - MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE); + public void testReAllocNullableVariableWidthVector() { + final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE); // Create a new value vector for 1024 integers - try (NullableVarCharVector vector = (NullableVarCharVector) TypeHelper.getNewVector(field, allocator)) { - NullableVarCharVector.Mutator m = vector.getMutator(); + try (final NullableVarCharVector vector = (NullableVarCharVector) TypeHelper.getNewVector(field, allocator)) { + final NullableVarCharVector.Mutator m = vector.getMutator(); vector.allocateNew(); int initialCapacity = vector.getValueCapacity(); @@ -439,9 +620,10 @@ public class TestValueVector extends ExecTest { // Check valueCapacity is more than initial allocation assertEquals((initialCapacity + 1) * 2 - 1, vector.getValueCapacity()); - assertArrayEquals(STR1, vector.getAccessor().get(0)); - assertArrayEquals(STR2, vector.getAccessor().get(initialCapacity - 1)); - assertArrayEquals(STR3, vector.getAccessor().get(initialCapacity + 200)); + final NullableVarCharVector.Accessor accessor = vector.getAccessor(); + assertArrayEquals(STR1, accessor.get(0)); + assertArrayEquals(STR2, accessor.get(initialCapacity - 1)); + assertArrayEquals(STR3, accessor.get(initialCapacity + 200)); // Set the valueCount to be more than valueCapacity of current allocation. This is possible for NullableValueVectors // as we don't call setSafe for null values, but we do call setValueCount when the current batch is processed.
