http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java index f2f97b7..e637518 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java @@ -1,17 +1,15 @@ package org.apache.drill.exec.store; -import com.carrotsearch.hppc.IntObjectOpenHashMap; -import com.carrotsearch.hppc.cursors.IntObjectCursor; -import com.carrotsearch.hppc.cursors.ObjectCursor; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; -import com.google.common.base.Charsets; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.common.io.Files; -import com.google.common.io.InputSupplier; -import com.google.common.io.Resources; +import static com.fasterxml.jackson.core.JsonToken.END_ARRAY; +import static com.fasterxml.jackson.core.JsonToken.END_OBJECT; +import static com.fasterxml.jackson.core.JsonToken.FIELD_NAME; + +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.util.List; + import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.SchemaPath; @@ -21,17 +19,34 @@ import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.proto.SchemaDefProtos; import org.apache.drill.exec.record.MaterializedField; -import org.apache.drill.exec.record.vector.*; -import org.apache.drill.exec.schema.*; +import org.apache.drill.exec.schema.DiffSchema; +import org.apache.drill.exec.schema.Field; +import org.apache.drill.exec.schema.IdGenerator; +import org.apache.drill.exec.schema.ListSchema; +import org.apache.drill.exec.schema.NamedField; +import org.apache.drill.exec.schema.ObjectSchema; +import org.apache.drill.exec.schema.OrderedField; +import org.apache.drill.exec.schema.RecordSchema; +import org.apache.drill.exec.schema.SchemaIdGenerator; import org.apache.drill.exec.schema.json.jackson.JacksonHelper; +import org.apache.drill.exec.vector.NullableBitVector; +import org.apache.drill.exec.vector.NullableFloat4Vector; +import org.apache.drill.exec.vector.NullableIntVector; +import org.apache.drill.exec.vector.NullableVarChar4Vector; +import org.apache.drill.exec.vector.TypeHelper; +import org.apache.drill.exec.vector.ValueVector; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.Charset; -import java.util.List; - -import static com.fasterxml.jackson.core.JsonToken.*; +import com.carrotsearch.hppc.IntObjectOpenHashMap; +import com.carrotsearch.hppc.cursors.ObjectCursor; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.google.common.base.Charsets; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.io.Files; +import com.google.common.io.InputSupplier; +import com.google.common.io.Resources; public class JSONRecordReader implements RecordReader { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JSONRecordReader.class); @@ -341,42 +356,40 @@ public class JSONRecordReader implements RecordReader { switch (minorType) { case INT: { holder.incAndCheckLength(32); - ValueVector.NullableInt int4 = (ValueVector.NullableInt) holder.getValueVector(); - if (val == null) { - int4.setNull(index); - } else { - int4.set(index, (Integer) val); + NullableIntVector int4 = (NullableIntVector) holder.getValueVector(); + NullableIntVector.Mutator m = int4.getMutator(); + if (val != null) { + m.set(index, (Integer) val); } return holder.hasEnoughSpace(32); } case FLOAT4: { holder.incAndCheckLength(32); - ValueVector.NullableFloat4 float4 = (ValueVector.NullableFloat4) holder.getValueVector(); - if (val == null) { - float4.setNull(index); - } else { - float4.set(index, (Float) val); + NullableFloat4Vector float4 = (NullableFloat4Vector) holder.getValueVector(); + NullableFloat4Vector.Mutator m = float4.getMutator(); + if (val != null) { + m.set(index, (Float) val); } return holder.hasEnoughSpace(32); } case VARCHAR4: { if (val == null) { - ((ValueVector.NullableVarChar4) holder.getValueVector()).setNull(index); return (index + 1) * 4 <= holder.getLength(); } else { byte[] bytes = ((String) val).getBytes(UTF_8); int length = bytes.length; holder.incAndCheckLength(length); - ValueVector.NullableVarChar4 varLen4 = (ValueVector.NullableVarChar4) holder.getValueVector(); - varLen4.set(index, bytes); + NullableVarChar4Vector varLen4 = (NullableVarChar4Vector) holder.getValueVector(); + NullableVarChar4Vector.Mutator m = varLen4.getMutator(); + m.set(index, bytes); return holder.hasEnoughSpace(length); } } case BOOLEAN: { holder.incAndCheckLength(1); - ValueVector.NullableBit bit = (ValueVector.NullableBit) holder.getValueVector(); + NullableBitVector bit = (NullableBitVector) holder.getValueVector(); if (val != null) { - bit.set(index, (Boolean)val ? 1 : 0); + bit.getMutator().set(index, (Boolean)val ? 1 : 0); } return holder.hasEnoughSpace(1); } @@ -409,7 +422,7 @@ public class JSONRecordReader implements RecordReader { SchemaDefProtos.MajorType type = field.getFieldType(); int fieldId = field.getFieldId(); MaterializedField f = MaterializedField.create(new SchemaPath(field.getFieldName()), fieldId, parentFieldId, type); - ValueVector.Base v = TypeHelper.getNewVector(f, allocator); + ValueVector v = TypeHelper.getNewVector(f, allocator); v.allocateNew(batchSize); VectorHolder holder = new VectorHolder(batchSize, v); valueVectorMap.put(fieldId, holder);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java index 4043913..fa0cbd5 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java @@ -18,22 +18,19 @@ package org.apache.drill.exec.store; -import org.apache.drill.exec.memory.BufferAllocator; -import org.apache.drill.exec.record.MaterializedField; -import org.apache.drill.exec.record.vector.TypeHelper; -import org.apache.drill.exec.record.vector.ValueVector; +import org.apache.drill.exec.vector.ValueVector; public class VectorHolder { private int length; - private ValueVector.Base vector; + private ValueVector vector; private int currentLength; - VectorHolder(int length, ValueVector.Base vector) { + VectorHolder(int length, ValueVector vector) { this.length = length; this.vector = vector; } - public ValueVector.Base getValueVector() { + public ValueVector getValueVector() { return vector; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java new file mode 100644 index 0000000..d18a29d --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java @@ -0,0 +1,123 @@ +package org.apache.drill.exec.vector; + +import java.util.Random; + +import org.apache.drill.exec.memory.BufferAllocator; +import org.apache.drill.exec.record.DeadBuf; +import org.apache.drill.exec.record.MaterializedField; +/** + * Bit implements a vector of bit-width values. Elements in the vector are accessed + * by position from the logical start of the vector. + * The width of each element is 1 bit. + * The equivalent Java primitive is an int containing the value '0' or '1'. + * + * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker. + */ +public final class BitVector extends ValueVector { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BitVector.class); + + public BitVector(MaterializedField field, BufferAllocator allocator) { + super(field, allocator); + } + + /** + * Get the byte holding the desired bit, then mask all other bits. Iff the result is 0, the + * bit was not set. + * + * @param index position of the bit in the vector + * @return 1 if set, otherwise 0 + */ + public int get(int index) { + // logger.debug("BIT GET: index: {}, byte: {}, mask: {}, masked byte: {}", + // index, + // data.getByte((int)Math.floor(index/8)), + // (int)Math.pow(2, (index % 8)), + // data.getByte((int)Math.floor(index/8)) & (int)Math.pow(2, (index % 8))); + return ((data.getByte((int)Math.floor(index/8)) & (int)Math.pow(2, (index % 8))) == 0) ? 0 : 1; + } + + @Override + public Object getObject(int index) { + return new Boolean(get(index) != 0); + } + + /** + * Get the size requirement (in bytes) for the given number of values. + */ + @Override + public int getSizeFromCount(int valueCount) { + return (int) Math.ceil(valueCount / 8); + } + + @Override + public int getAllocatedSize() { + return totalBytes; + } + + public Mutator getMutator() { + return new Mutator(); + } + + /** + * Allocate a new memory space for this vector. Must be called prior to using the ValueVector. + * + * @param valueCount The number of values which can be contained within this vector. + */ + @Override + public void allocateNew(int valueCount) { + allocateNew(getSizeFromCount(valueCount), null, valueCount); + for (int i = 0; i < getSizeFromCount(valueCount); i++) { + data.setByte(i, 0); + } + } + + + /** + * MutableBit implements a vector of bit-width values. Elements in the vector are accessed + * by position from the logical start of the vector. Values should be pushed onto the vector + * sequentially, but may be randomly accessed. + * + * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker. + */ + public class Mutator implements ValueVector.Mutator{ + + private Mutator(){} + + /** + * Set the bit at the given index to the specified value. + * + * @param index position of the bit to set + * @param value value to set (either 1 or 0) + */ + public void set(int index, int value) { + byte currentByte = data.getByte((int)Math.floor(index/8)); + if (value != 0) { + // true + currentByte |= (byte) Math.pow(2, (index % 8)); + } + else if ((currentByte & (byte) Math.pow(2, (index % 8))) == (byte) Math.pow(2, (index % 8))) { + // false, and bit was previously set + currentByte -= (byte) Math.pow(2, (index % 8)); + } + data.setByte((int) Math.floor(index/8), currentByte); + } + + + @Override + public void setRecordCount(int recordCount) { + BitVector.this.setRecordCount(recordCount); + } + + @Override + public void randomizeData() { + if (data != DeadBuf.DEAD_BUFFER) { + Random r = new Random(); + for (int i = 0; i < data.capacity() - 1; i++) { + byte[] bytes = new byte[1]; + r.nextBytes(bytes); + data.setByte(i, bytes[0]); + } + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java new file mode 100644 index 0000000..718478e --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java @@ -0,0 +1,192 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.exec.vector; + +import io.netty.buffer.ByteBuf; + +import java.io.Closeable; + +import org.apache.drill.exec.memory.BufferAllocator; +import org.apache.drill.exec.proto.UserBitShared.FieldMetadata; +import org.apache.drill.exec.record.DeadBuf; +import org.apache.drill.exec.record.MaterializedField; + +/** + * ValueVectorTypes defines a set of template-generated classes which implement type-specific + * value vectors. The template approach was chosen due to the lack of multiple inheritence. It + * is also important that all related logic be as efficient as possible. + */ +public abstract class ValueVector implements Closeable { + + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueVector.class); + + protected final BufferAllocator allocator; + protected ByteBuf data = DeadBuf.DEAD_BUFFER; + protected MaterializedField field; + protected int recordCount; + protected int totalBytes; + + ValueVector(MaterializedField field, BufferAllocator allocator) { + this.allocator = allocator; + this.field = field; + } + + /** + * Get the explicitly specified size of the allocated buffer, if available. Otherwise + * calculate the size based on width and record count. + */ + public abstract int getAllocatedSize(); + + /** + * Get the size requirement (in bytes) for the given number of values. Takes derived + * type specs into account. + */ + public abstract int getSizeFromCount(int valueCount); + + /** + * Get the Java Object representation of the element at the specified position + * + * @param index Index of the value to get + */ + public abstract Object getObject(int index); + + + public abstract Mutator getMutator(); + + /** + * Return the underlying buffers associated with this vector. Note that this doesn't impact the + * reference counts for this buffer so it only should be used for in-context access. Also note + * that this buffer changes regularly thus external classes shouldn't hold a reference to + * it (unless they change it). + * + * @return The underlying ByteBuf. + */ + public ByteBuf[] getBuffers() { + return new ByteBuf[]{data}; + } + + /** + * Returns the maximum number of values contained within this vector. + * @return Vector size + */ + public int capacity() { + return getRecordCount(); + } + + /** + * Release supporting resources. + */ + @Override + public void close() { + clear(); + } + + /** + * Get information about how this field is materialized. + * @return + */ + public MaterializedField getField() { + return field; + } + + /** + * Get the number of records allocated for this value vector. + * @return number of allocated records + */ + public int getRecordCount() { + return recordCount; + } + + /** + * Get the metadata for this field. + * @return + */ + public FieldMetadata getMetadata() { + int len = 0; + for(ByteBuf b : getBuffers()){ + len += b.writerIndex(); + } + return FieldMetadata.newBuilder() + .setDef(getField().getDef()) + .setValueCount(getRecordCount()) + .setBufferLength(len) + .build(); + } + + /** + * Allocate a new memory space for this vector. Must be called prior to using the ValueVector. + * + * @param totalBytes Optional desired size of the underlying buffer. Specifying 0 will + * estimate the size based on valueCount. + * @param sourceBuffer Optional ByteBuf to use for storage (null will allocate automatically). + * @param valueCount Number of values in the vector. + */ + public void allocateNew(int totalBytes, ByteBuf sourceBuffer, int valueCount) { + clear(); + this.recordCount = valueCount; + this.totalBytes = totalBytes > 0 ? totalBytes : getSizeFromCount(valueCount); + this.data = (sourceBuffer != null) ? sourceBuffer : allocator.buffer(this.totalBytes); + this.data.retain(); + data.readerIndex(0); + } + + /** + * Allocate a new memory space for this vector. Must be called prior to using the ValueVector. + * + * @param valueCount + * The number of elements which can be contained within this vector. + */ + public void allocateNew(int valueCount) { + allocateNew(0, null, valueCount); + } + + /** + * Release the underlying ByteBuf and reset the ValueVector + */ + protected void clear() { + if (data != DeadBuf.DEAD_BUFFER) { + data.release(); + data = DeadBuf.DEAD_BUFFER; + recordCount = 0; + totalBytes = 0; + } + } + + //public abstract <T extends Mutator> T getMutator(); + + /** + * Define the number of records that are in this value vector. + * @param recordCount Number of records active in this vector. + */ + void setRecordCount(int recordCount) { + data.writerIndex(getSizeFromCount(recordCount)); + this.recordCount = recordCount; + } + + /** + * For testing only -- randomize the buffer contents + */ + public void randomizeData() { } + + + public static interface Mutator{ + public void randomizeData(); + public void setRecordCount(int recordCount); + } +} + http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java index 3edf283..3fe0622 100644 --- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java +++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java @@ -26,10 +26,10 @@ import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.pop.PopUnitTestBase; import org.apache.drill.exec.proto.UserProtos.QueryType; import org.apache.drill.exec.record.RecordBatchLoader; -import org.apache.drill.exec.record.vector.ValueVector; import org.apache.drill.exec.rpc.user.QueryResultBatch; import org.apache.drill.exec.server.Drillbit; import org.apache.drill.exec.server.RemoteServiceSet; +import org.apache.drill.exec.vector.ValueVector; import org.junit.Test; import com.carrotsearch.hppc.cursors.IntObjectCursor; @@ -61,7 +61,7 @@ public class TestSimpleFragmentRun extends PopUnitTestBase { // print headers. if (schemaChanged) { System.out.println("\n\n========NEW SCHEMA=========\n\n"); - for (IntObjectCursor<ValueVector.Base> v : batchLoader) { + for (IntObjectCursor<ValueVector> v : batchLoader) { if (firstColumn) { firstColumn = false; @@ -80,7 +80,7 @@ public class TestSimpleFragmentRun extends PopUnitTestBase { for (int i = 0; i < batchLoader.getRecordCount(); i++) { boolean first = true; recordCount++; - for (IntObjectCursor<ValueVector.Base> v : batchLoader) { + for (IntObjectCursor<ValueVector> v : batchLoader) { if (first) { first = false; } else { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java index 5924f7d..ae4f644 100644 --- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java +++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java @@ -1,19 +1,21 @@ package org.apache.drill.exec.record.vector; -import io.netty.buffer.ByteBuf; -import org.apache.drill.exec.memory.DirectBufferAllocator; -import org.apache.drill.exec.proto.SchemaDefProtos; -import org.apache.drill.exec.record.MaterializedField; - -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import org.apache.hadoop.io.UTF8; -import org.junit.Test; - import java.nio.charset.Charset; +import org.apache.drill.exec.memory.DirectBufferAllocator; +import org.apache.drill.exec.proto.SchemaDefProtos; +import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.vector.BitVector; +import org.apache.drill.exec.vector.NullableFloat4Vector; +import org.apache.drill.exec.vector.NullableUInt4Vector; +import org.apache.drill.exec.vector.NullableVarChar2Vector; +import org.apache.drill.exec.vector.TypeHelper; +import org.apache.drill.exec.vector.UInt4Vector; +import org.junit.Test; + public class TestValueVector { DirectBufferAllocator allocator = new DirectBufferAllocator(); @@ -34,15 +36,16 @@ public class TestValueVector { MaterializedField field = MaterializedField.create(defBuilder.build()); // Create a new value vector for 1024 integers - ValueVector.MutableUInt4 v = new ValueVector.MutableUInt4(field, allocator); + UInt4Vector v = new UInt4Vector(field, allocator); + UInt4Vector.Mutator m = v.getMutator(); v.allocateNew(1024); // Put and set a few values - v.set(0, 100); - v.set(1, 101); - v.set(100, 102); - v.set(1022, 103); - v.set(1023, 104); + m.set(0, 100); + m.set(1, 101); + m.set(100, 102); + m.set(1022, 103); + m.set(1023, 104); assertEquals(100, v.get(0)); assertEquals(101, v.get(1)); assertEquals(102, v.get(100)); @@ -69,16 +72,17 @@ public class TestValueVector { MaterializedField field = MaterializedField.create(defBuilder.build()); // Create a new value vector for 1024 integers - ValueVector.NullableVarChar2 v = new ValueVector.NullableVarChar2(field, allocator); + NullableVarChar2Vector v = new NullableVarChar2Vector(field, allocator); + NullableVarChar2Vector.Mutator m = v.getMutator(); v.allocateNew(1024); // Create and set 3 sample strings String str1 = new String("AAAAA1"); String str2 = new String("BBBBBBBBB2"); String str3 = new String("CCCC3"); - v.set(0, str1.getBytes(Charset.forName("UTF-8"))); - v.set(1, str2.getBytes(Charset.forName("UTF-8"))); - v.set(2, str3.getBytes(Charset.forName("UTF-8"))); + m.set(0, str1.getBytes(Charset.forName("UTF-8"))); + m.set(1, str2.getBytes(Charset.forName("UTF-8"))); + m.set(2, str3.getBytes(Charset.forName("UTF-8"))); // Check the sample strings assertEquals(str1, new String(v.get(0), Charset.forName("UTF-8"))); @@ -86,10 +90,16 @@ public class TestValueVector { assertEquals(str3, new String(v.get(2), Charset.forName("UTF-8"))); // Ensure null value throws + boolean b = false; try { v.get(3); - assertFalse(false); - } catch(NullValueException e) { } + } catch(AssertionError e) { + b = true; + }finally{ + if(!b){ + assert false; + } + } } @@ -110,15 +120,16 @@ public class TestValueVector { MaterializedField field = MaterializedField.create(defBuilder.build()); // Create a new value vector for 1024 integers - ValueVector.NullableUInt4 v = new ValueVector.NullableUInt4(field, allocator); + NullableUInt4Vector v = new NullableUInt4Vector(field, allocator); + NullableUInt4Vector.Mutator m = v.getMutator(); v.allocateNew(1024); // Put and set a few values - v.set(0, 100); - v.set(1, 101); - v.set(100, 102); - v.set(1022, 103); - v.set(1023, 104); + m.set(0, 100); + m.set(1, 101); + m.set(100, 102); + m.set(1022, 103); + m.set(1023, 104); assertEquals(100, v.get(0)); assertEquals(101, v.get(1)); assertEquals(102, v.get(100)); @@ -126,22 +137,39 @@ public class TestValueVector { assertEquals(104, v.get(1023)); // Ensure null values throw - try { - v.get(3); - assertFalse(false); - } catch(NullValueException e) { } - + { + boolean b = false; + try { + v.get(3); + } catch(AssertionError e) { + b = true; + }finally{ + if(!b){ + assert false; + } + } + } + + v.allocateNew(2048); - try { - v.get(0); - assertFalse(false); - } catch(NullValueException e) { } - - v.set(0, 100); - v.set(1, 101); - v.set(100, 102); - v.set(1022, 103); - v.set(1023, 104); + { + boolean b = false; + try { + v.get(0); + } catch(AssertionError e) { + b = true; + }finally{ + if(!b){ + assert false; + } + } + } + + m.set(0, 100); + m.set(1, 101); + m.set(100, 102); + m.set(1022, 103); + m.set(1023, 104); assertEquals(100, v.get(0)); assertEquals(101, v.get(1)); assertEquals(102, v.get(100)); @@ -149,10 +177,19 @@ public class TestValueVector { assertEquals(104, v.get(1023)); // Ensure null values throw - try { - v.get(3); - assertFalse(false); - } catch(NullValueException e) { } + + { + boolean b = false; + try { + v.get(3); + } catch(AssertionError e) { + b = true; + }finally{ + if(!b){ + assert false; + } + } + } } @@ -172,16 +209,16 @@ public class TestValueVector { MaterializedField field = MaterializedField.create(defBuilder.build()); // Create a new value vector for 1024 integers - ValueVector.NullableFloat4 v = (ValueVector.NullableFloat4) TypeHelper.getNewVector(field, allocator); - + NullableFloat4Vector v = (NullableFloat4Vector) TypeHelper.getNewVector(field, allocator); + NullableFloat4Vector.Mutator m = v.getMutator(); v.allocateNew(1024); // Put and set a few values - v.set(0, 100.1f); - v.set(1, 101.2f); - v.set(100, 102.3f); - v.set(1022, 103.4f); - v.set(1023, 104.5f); + m.set(0, 100.1f); + m.set(1, 101.2f); + m.set(100, 102.3f); + m.set(1022, 103.4f); + m.set(1023, 104.5f); assertEquals(100.1f, v.get(0), 0); assertEquals(101.2f, v.get(1), 0); assertEquals(102.3f, v.get(100), 0); @@ -189,17 +226,32 @@ public class TestValueVector { assertEquals(104.5f, v.get(1023), 0); // Ensure null values throw - try { - v.get(3); - assertFalse(false); - } catch(NullValueException e) { } - + { + boolean b = false; + try { + v.get(3); + } catch(AssertionError e) { + b = true; + }finally{ + if(!b){ + assert false; + } + } + } + v.allocateNew(2048); - try { - v.get(0); - assertFalse(false); - } catch(NullValueException e) { } - + { + boolean b = false; + try { + v.get(0); + } catch(AssertionError e) { + b = true; + }finally{ + if(!b){ + assert false; + } + } + } } @Test @@ -218,30 +270,31 @@ public class TestValueVector { MaterializedField field = MaterializedField.create(defBuilder.build()); // Create a new value vector for 1024 integers - ValueVector.MutableBit v = new ValueVector.MutableBit(field, allocator); + BitVector v = new BitVector(field, allocator); + BitVector.Mutator m = v.getMutator(); v.allocateNew(1024); // Put and set a few values - v.set(0, 1); - v.set(1, 0); - v.set(100, 0); - v.set(1022, 1); + m.set(0, 1); + m.set(1, 0); + m.set(100, 0); + m.set(1022, 1); assertEquals(1, v.get(0)); assertEquals(0, v.get(1)); assertEquals(0, v.get(100)); assertEquals(1, v.get(1022)); // test setting the same value twice - v.set(0, 1); - v.set(0, 1); - v.set(1, 0); - v.set(1, 0); + m.set(0, 1); + m.set(0, 1); + m.set(1, 0); + m.set(1, 0); assertEquals(1, v.get(0)); assertEquals(0, v.get(1)); // test toggling the values - v.set(0, 0); - v.set(1, 1); + m.set(0, 0); + m.set(1, 1); assertEquals(0, v.get(0)); assertEquals(1, v.get(1)); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java index cef40ff..117414c 100644 --- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java +++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java @@ -1,8 +1,16 @@ package org.apache.drill.exec.store; -import com.beust.jcommander.internal.Lists; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.List; + import mockit.Expectations; import mockit.Injectable; + import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.memory.DirectBufferAllocator; @@ -10,17 +18,11 @@ import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.proto.SchemaDefProtos; import org.apache.drill.exec.proto.UserBitShared; -import org.apache.drill.exec.record.vector.ValueVector; +import org.apache.drill.exec.vector.ValueVector; import org.junit.Ignore; import org.junit.Test; -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.Arrays; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import com.beust.jcommander.internal.Lists; public class JSONRecordReaderTest { private static final Charset UTF_8 = Charset.forName("UTF-8"); @@ -31,7 +33,7 @@ public class JSONRecordReaderTest { class MockOutputMutator implements OutputMutator { List<Integer> removedFields = Lists.newArrayList(); - List<ValueVector.Base> addFields = Lists.newArrayList(); + List<ValueVector> addFields = Lists.newArrayList(); @Override public void removeField(int fieldId) throws SchemaChangeException { @@ -39,7 +41,7 @@ public class JSONRecordReaderTest { } @Override - public void addField(int fieldId, ValueVector.Base vector) throws SchemaChangeException { + public void addField(int fieldId, ValueVector vector) throws SchemaChangeException { addFields.add(vector); } @@ -51,16 +53,16 @@ public class JSONRecordReaderTest { return removedFields; } - List<ValueVector.Base> getAddFields() { + List<ValueVector> getAddFields() { return addFields; } } - private <T> void assertField(ValueVector.Base valueVector, int index, SchemaDefProtos.MinorType expectedMinorType, T value, String name) { + private <T> void assertField(ValueVector valueVector, int index, SchemaDefProtos.MinorType expectedMinorType, T value, String name) { assertField(valueVector, index, expectedMinorType, value, name, 0); } - private <T> void assertField(ValueVector.Base valueVector, int index, SchemaDefProtos.MinorType expectedMinorType, T value, String name, int parentFieldId) { + private <T> void assertField(ValueVector valueVector, int index, SchemaDefProtos.MinorType expectedMinorType, T value, String name, int parentFieldId) { UserBitShared.FieldMetadata metadata = valueVector.getMetadata(); SchemaDefProtos.FieldDef def = metadata.getDef(); assertEquals(expectedMinorType, def.getMajorType().getMinorType()); @@ -90,7 +92,7 @@ public class JSONRecordReaderTest { JSONRecordReader jr = new JSONRecordReader(context, getResource("scan_json_test_1.json")); MockOutputMutator mutator = new MockOutputMutator(); - List<ValueVector.Base> addFields = mutator.getAddFields(); + List<ValueVector> addFields = mutator.getAddFields(); jr.setup(mutator); assertEquals(2, jr.next()); assertEquals(3, addFields.size()); @@ -116,7 +118,7 @@ public class JSONRecordReaderTest { JSONRecordReader jr = new JSONRecordReader(context, getResource("scan_json_test_2.json")); MockOutputMutator mutator = new MockOutputMutator(); - List<ValueVector.Base> addFields = mutator.getAddFields(); + List<ValueVector> addFields = mutator.getAddFields(); jr.setup(mutator); assertEquals(3, jr.next()); @@ -142,7 +144,7 @@ public class JSONRecordReaderTest { assertEquals(0, jr.next()); } - @Test + @Test @Ignore public void testChangedSchemaInTwoBatches(@Injectable final FragmentContext context) throws IOException, ExecutionSetupException { new Expectations() { { @@ -153,7 +155,7 @@ public class JSONRecordReaderTest { JSONRecordReader jr = new JSONRecordReader(context, getResource("scan_json_test_2.json"), 64); // batch only fits 1 int MockOutputMutator mutator = new MockOutputMutator(); - List<ValueVector.Base> addFields = mutator.getAddFields(); + List<ValueVector> addFields = mutator.getAddFields(); List<Integer> removedFields = mutator.getRemovedFields(); jr.setup(mutator); @@ -201,7 +203,7 @@ public class JSONRecordReaderTest { JSONRecordReader jr = new JSONRecordReader(context, getResource("scan_json_test_3.json")); MockOutputMutator mutator = new MockOutputMutator(); - List<ValueVector.Base> addFields = mutator.getAddFields(); + List<ValueVector> addFields = mutator.getAddFields(); jr.setup(mutator); assertEquals(2, jr.next()); assertEquals(5, addFields.size());
