DRILL-6335: Column accessor refactoring closes #1218
Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/dbff1646 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/dbff1646 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/dbff1646 Branch: refs/heads/master Commit: dbff1646601db234a6606d400d5630db4deee192 Parents: 883c8d9 Author: Paul Rogers <[email protected]> Authored: Mon Apr 16 21:44:10 2018 -0700 Committer: Vitalii Diravka <[email protected]> Committed: Sun Apr 29 23:20:55 2018 +0300 ---------------------------------------------------------------------- .../exec/physical/rowSet/impl/ColumnState.java | 10 +- .../rowSet/impl/PrimitiveColumnState.java | 9 +- .../rowSet/impl/RepeatedVectorState.java | 3 +- .../rowSet/impl/ResultSetLoaderImpl.java | 2 +- .../physical/rowSet/impl/RowSetLoaderImpl.java | 9 + .../physical/rowSet/impl/SingleVectorState.java | 11 +- .../exec/physical/rowSet/impl/TupleState.java | 25 +- .../rowSet/model/single/BaseWriterBuilder.java | 3 +- .../impl/TestResultSetLoaderMapArray.java | 3 +- .../rowSet/impl/TestResultSetLoaderMaps.java | 35 +-- .../impl/TestResultSetLoaderProjection.java | 4 +- .../impl/TestResultSetLoaderProtocol.java | 20 +- .../drill/test/rowSet/RowSetUtilities.java | 2 +- .../drill/test/rowSet/RowSetWriterImpl.java | 9 + .../drill/test/rowSet/test/DummyWriterTest.java | 22 +- .../drill/test/rowSet/test/PerformanceTool.java | 3 +- .../test/rowSet/test/TestIndirectReaders.java | 151 ++++++++++ .../rowSet/test/TestOffsetVectorWriter.java | 28 +- .../main/codegen/templates/ColumnAccessors.java | 65 ++-- .../exec/record/metadata/ProjectionType.java | 2 +- .../drill/exec/vector/accessor/ArrayWriter.java | 73 +++-- .../exec/vector/accessor/ColumnWriter.java | 128 ++++++++ .../exec/vector/accessor/ObjectWriter.java | 53 +--- .../exec/vector/accessor/ScalarWriter.java | 9 +- .../drill/exec/vector/accessor/TupleWriter.java | 52 ++-- .../exec/vector/accessor/WriterPosition.java | 58 ++++ .../accessor/writer/AbstractArrayWriter.java | 196 ++++++------ .../writer/AbstractFixedWidthWriter.java | 18 +- .../accessor/writer/AbstractObjectWriter.java | 45 ++- .../accessor/writer/AbstractScalarWriter.java | 59 +++- .../accessor/writer/AbstractTupleWriter.java | 115 +++---- .../accessor/writer/BaseScalarWriter.java | 40 +-- .../accessor/writer/BaseVarWidthWriter.java | 6 +- .../accessor/writer/ColumnWriterFactory.java | 111 +++---- .../exec/vector/accessor/writer/MapWriter.java | 92 +++++- .../accessor/writer/NullableScalarWriter.java | 16 +- .../accessor/writer/ObjectArrayWriter.java | 44 ++- .../accessor/writer/OffsetVectorWriter.java | 277 +---------------- .../accessor/writer/OffsetVectorWriterImpl.java | 299 +++++++++++++++++++ .../accessor/writer/ScalarArrayWriter.java | 111 +++++-- .../vector/accessor/writer/WriterEvents.java | 45 ++- .../accessor/writer/dummy/DummyArrayWriter.java | 44 +-- .../writer/dummy/DummyScalarWriter.java | 16 +- .../vector/accessor/writer/package-info.java | 97 +++++- 44 files changed, 1505 insertions(+), 915 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ColumnState.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ColumnState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ColumnState.java index 33d3ffe..7fae5ad 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ColumnState.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ColumnState.java @@ -28,8 +28,9 @@ import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; import org.apache.drill.exec.vector.accessor.writer.AbstractArrayWriter; import org.apache.drill.exec.vector.accessor.writer.AbstractObjectWriter; -import org.apache.drill.exec.vector.accessor.writer.ColumnWriterFactory; +import org.apache.drill.exec.vector.accessor.writer.MapWriter; import org.apache.drill.exec.vector.complex.BaseRepeatedValueVector; +import org.apache.drill.exec.vector.complex.MapVector; /** * Represents the write-time state for a column including the writer and the (optional) @@ -88,10 +89,10 @@ public abstract class ColumnState { public static class MapColumnState extends BaseMapColumnState { public MapColumnState(ResultSetLoaderImpl resultSetLoader, - ColumnMetadata columnSchema, + ColumnMetadata columnSchema, MapVector mapVector, ProjectionSet projectionSet) { super(resultSetLoader, - ColumnWriterFactory.buildMap(columnSchema, null, + MapWriter.buildMap(columnSchema, mapVector, new ArrayList<AbstractObjectWriter>()), new NullVectorState(), projectionSet); @@ -115,7 +116,6 @@ public abstract class ColumnState { projectionSet); } - @SuppressWarnings("resource") public static MapArrayColumnState build(ResultSetLoaderImpl resultSetLoader, ColumnMetadata columnSchema, ProjectionSet projectionSet) { @@ -128,7 +128,7 @@ public abstract class ColumnState { // Create the writer using the offset vector - AbstractObjectWriter writer = ColumnWriterFactory.buildMapArray( + AbstractObjectWriter writer = MapWriter.buildMapArray( columnSchema, offsetVector, new ArrayList<AbstractObjectWriter>()); http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/PrimitiveColumnState.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/PrimitiveColumnState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/PrimitiveColumnState.java index c97ec18..55ccb74 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/PrimitiveColumnState.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/PrimitiveColumnState.java @@ -20,6 +20,7 @@ package org.apache.drill.exec.physical.rowSet.impl; import org.apache.drill.exec.physical.rowSet.impl.SingleVectorState.ValuesVectorState; import org.apache.drill.exec.vector.NullableVector; import org.apache.drill.exec.vector.ValueVector; +import org.apache.drill.exec.vector.accessor.ObjectType; import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.apache.drill.exec.vector.accessor.ScalarWriter.ColumnWriterListener; import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; @@ -38,7 +39,13 @@ public class PrimitiveColumnState extends ColumnState implements ColumnWriterLis AbstractObjectWriter colWriter, VectorState vectorState) { super(resultSetLoader, colWriter, vectorState); - writer.bindListener(this); + ScalarWriter scalarWriter; + if (colWriter.type() == ObjectType.ARRAY) { + scalarWriter = writer.array().scalar(); + } else { + scalarWriter = writer.scalar(); + } + ((AbstractScalarWriter) scalarWriter).bindListener(this); } public static PrimitiveColumnState newPrimitive( http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RepeatedVectorState.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RepeatedVectorState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RepeatedVectorState.java index 9bd1ef2..afc1cac 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RepeatedVectorState.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RepeatedVectorState.java @@ -56,7 +56,8 @@ public class RepeatedVectorState implements VectorState { // Create the offsets state with the offset vector portion of the repeated // vector, and the offset writer portion of the array writer. - offsetsState = new OffsetVectorState(arrayWriter.offsetWriter(), + offsetsState = new OffsetVectorState( + arrayWriter.offsetWriter(), vector.getOffsetVector(), (AbstractObjectWriter) arrayWriter.entry()); } http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ResultSetLoaderImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ResultSetLoaderImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ResultSetLoaderImpl.java index cc50729..c386979 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ResultSetLoaderImpl.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ResultSetLoaderImpl.java @@ -401,7 +401,7 @@ public class ResultSetLoaderImpl implements ResultSetLoader { @Override public ResultSetLoader setRow(Object... values) { startRow(); - writer().setTuple(values); + writer().setObject(values); saveRow(); return this; } http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RowSetLoaderImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RowSetLoaderImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RowSetLoaderImpl.java index dca749c..68edbee 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RowSetLoaderImpl.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RowSetLoaderImpl.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import org.apache.drill.exec.physical.rowSet.ResultSetLoader; import org.apache.drill.exec.physical.rowSet.RowSetLoader; +import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.accessor.writer.AbstractObjectWriter; import org.apache.drill.exec.vector.accessor.writer.AbstractTupleWriter; @@ -95,4 +96,12 @@ public class RowSetLoaderImpl extends AbstractTupleWriter implements RowSetLoade @Override public int rowCount() { return rsLoader.rowCount(); } + + @Override + public ColumnMetadata schema() { + // The top-level tuple (the data row) is not associated + // with a parent column. By contrast, a map tuple is + // associated with the column that defines the map. + return null; + } } http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/SingleVectorState.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/SingleVectorState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/SingleVectorState.java index e813a70..0efb6f1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/SingleVectorState.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/SingleVectorState.java @@ -23,6 +23,7 @@ import org.apache.drill.exec.vector.FixedWidthVector; import org.apache.drill.exec.vector.UInt4Vector; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.VariableWidthVector; +import org.apache.drill.exec.vector.accessor.WriterPosition; import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; import org.apache.drill.exec.vector.accessor.writer.AbstractObjectWriter; import org.apache.drill.exec.vector.accessor.writer.AbstractScalarWriter; @@ -98,7 +99,7 @@ public abstract class SingleVectorState implements VectorState { private final AbstractObjectWriter childWriter; - public OffsetVectorState(AbstractScalarWriter writer, ValueVector mainVector, + public OffsetVectorState(WriterPosition writer, ValueVector mainVector, AbstractObjectWriter childWriter) { super(writer, mainVector); this.childWriter = childWriter; @@ -145,7 +146,7 @@ public abstract class SingleVectorState implements VectorState { UInt4Vector.Accessor sourceAccessor = ((UInt4Vector) backupVector).getAccessor(); UInt4Vector.Mutator destMutator = ((UInt4Vector) mainVector).getMutator(); - int offset = childWriter.events().writerIndex().rowStartIndex(); + int offset = childWriter.rowStartIndex(); int newIndex = 1; ResultSetLoaderImpl.logger.trace("Offset vector: copy {} values from {} to {} with offset {}", Math.max(0, sourceEndIndex - sourceStartIndex + 1), @@ -163,11 +164,11 @@ public abstract class SingleVectorState implements VectorState { } } - protected final AbstractScalarWriter writer; + protected final WriterPosition writer; protected final ValueVector mainVector; protected ValueVector backupVector; - public SingleVectorState(AbstractScalarWriter writer, ValueVector mainVector) { + public SingleVectorState(WriterPosition writer, ValueVector mainVector) { this.writer = writer; this.mainVector = mainVector; } @@ -198,7 +199,7 @@ public abstract class SingleVectorState implements VectorState { @Override public void rollover(int cardinality) { - int sourceStartIndex = writer.writerIndex().rowStartIndex(); + int sourceStartIndex = writer.rowStartIndex(); // Remember the last write index for the original vector. // This tells us the end of the set of values to move, while the http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/TupleState.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/TupleState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/TupleState.java index 82f0437..bdf303c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/TupleState.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/TupleState.java @@ -27,6 +27,7 @@ import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.metadata.AbstractColumnMetadata; import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.record.metadata.MetadataUtils; +import org.apache.drill.exec.record.metadata.ProjectionType; import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.record.metadata.TupleSchema; import org.apache.drill.exec.vector.ValueVector; @@ -38,6 +39,7 @@ import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; import org.apache.drill.exec.vector.accessor.writer.AbstractObjectWriter; import org.apache.drill.exec.vector.accessor.writer.AbstractTupleWriter; import org.apache.drill.exec.vector.accessor.writer.ColumnWriterFactory; +import org.apache.drill.exec.vector.complex.MapVector; /** * Represents the loader state for a tuple: a row or a map. This is "state" in @@ -103,7 +105,11 @@ public abstract class TupleState implements TupleWriterListener { ProjectionSet projectionSet) { super(rsLoader, projectionSet); this.mapColumnState = mapColumnState; - mapColumnState.writer().bindListener(this); + if (mapColumnState.schema().isArray()) { + mapColumnState.writer().array().tuple().bindListener(this); + } else { + mapColumnState.writer().tuple().bindListener(this); + } } /** @@ -177,7 +183,7 @@ public abstract class TupleState implements TupleWriterListener { public List<ColumnState> columns() { return columns; } - public TupleMetadata schema() { return writer().schema(); } + public TupleMetadata schema() { return writer().tupleSchema(); } public abstract AbstractTupleWriter writer(); @@ -200,6 +206,13 @@ public abstract class TupleState implements TupleWriterListener { return addColumn(columnSchema); } + @Override + public ProjectionType projectionType(String columnName) { + return projectionSet.isProjected(columnName) ? + ProjectionType.TUPLE : + ProjectionType.UNPROJECTED; + } + /** * Implementation of the work to add a new column to this tuple given a * schema description of the column. @@ -291,8 +304,14 @@ public abstract class TupleState implements TupleWriterListener { columnSchema, childProjection); } else { + MapVector vector; + if (columnSchema.isProjected()) { + vector = new MapVector(columnSchema.schema(), resultSetLoader.allocator(), null); + } else { + vector = null; + } return new MapColumnState(resultSetLoader, - columnSchema, + columnSchema, vector, childProjection); } } http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/model/single/BaseWriterBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/model/single/BaseWriterBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/model/single/BaseWriterBuilder.java index bab7b39..a41c33d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/model/single/BaseWriterBuilder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/model/single/BaseWriterBuilder.java @@ -28,6 +28,7 @@ import org.apache.drill.exec.record.VectorContainer; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.writer.AbstractObjectWriter; import org.apache.drill.exec.vector.accessor.writer.ColumnWriterFactory; +import org.apache.drill.exec.vector.accessor.writer.MapWriter; import org.apache.drill.exec.vector.complex.AbstractMapVector; /** @@ -50,7 +51,7 @@ public abstract class BaseWriterBuilder { private AbstractObjectWriter buildVectorWriter(ValueVector vector, VectorDescrip descrip) { MajorType type = vector.getField().getType(); if (type.getMinorType() == MinorType.MAP) { - return ColumnWriterFactory.buildMapWriter(descrip.metadata, + return MapWriter.buildMapWriter(descrip.metadata, (AbstractMapVector) vector, buildMap((AbstractMapVector) vector, descrip)); } else { http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMapArray.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMapArray.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMapArray.java index a3f1754..3ed1dc2 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMapArray.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMapArray.java @@ -59,7 +59,6 @@ import org.junit.Test; public class TestResultSetLoaderMapArray extends SubOperatorTest { - @SuppressWarnings("resource") @Test public void testBasics() { TupleMetadata schema = new SchemaBuilder() @@ -77,7 +76,7 @@ public class TestResultSetLoaderMapArray extends SubOperatorTest { // Verify structure and schema - TupleMetadata actualSchema = rootWriter.schema(); + TupleMetadata actualSchema = rootWriter.tupleSchema(); assertEquals(2, actualSchema.size()); assertTrue(actualSchema.metadata(1).isArray()); assertTrue(actualSchema.metadata(1).isMap()); http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMaps.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMaps.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMaps.java index 10dbbe1..89a01e3 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMaps.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMaps.java @@ -19,7 +19,6 @@ package org.apache.drill.exec.physical.rowSet.impl; import static org.apache.drill.test.rowSet.RowSetUtilities.intArray; import static org.apache.drill.test.rowSet.RowSetUtilities.mapValue; -import static org.apache.drill.test.rowSet.RowSetUtilities.objArray; import static org.apache.drill.test.rowSet.RowSetUtilities.strArray; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -45,6 +44,7 @@ import org.apache.drill.test.rowSet.RowSet; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.apache.drill.test.rowSet.RowSetComparison; import org.apache.drill.test.rowSet.RowSetReader; +import org.apache.drill.test.rowSet.RowSetUtilities; import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.junit.Test; @@ -74,7 +74,7 @@ public class TestResultSetLoaderMaps extends SubOperatorTest { // Verify structure and schema assertEquals(5, rsLoader.schemaVersion()); - TupleMetadata actualSchema = rootWriter.schema(); + TupleMetadata actualSchema = rootWriter.tupleSchema(); assertEquals(3, actualSchema.size()); assertTrue(actualSchema.metadata(1).isMap()); assertEquals(2, actualSchema.metadata("m").mapSchema().size()); @@ -123,7 +123,7 @@ public class TestResultSetLoaderMaps extends SubOperatorTest { .addRow(20, mapValue(210, "barney"), "bam-bam") .build(); - new RowSetComparison(expected).verifyAndClearAll(actual); + RowSetUtilities.verify(expected, actual); rsLoader.close(); } @@ -164,7 +164,7 @@ public class TestResultSetLoaderMaps extends SubOperatorTest { .addRow(20, mapValue("barney")) .build(); - new RowSetComparison(expected).verifyAndClearAll(actual); + RowSetUtilities.verify(expected, actual); // Add three columns in the second batch. One before // the batch starts, one before the first row, and one after @@ -201,8 +201,7 @@ public class TestResultSetLoaderMaps extends SubOperatorTest { .addRow(40, mapValue("betty", 140, 140_000L, "bam-bam")) .build(); - new RowSetComparison(expected).verifyAndClearAll(actual); - + RowSetUtilities.verify(expected, actual); rsLoader.close(); } @@ -238,7 +237,7 @@ public class TestResultSetLoaderMaps extends SubOperatorTest { // Ensure metadata was added - assertTrue(mapWriter.schema().size() == 1); + assertTrue(mapWriter.tupleSchema().size() == 1); rootWriter .addRow(20, mapValue("fred")) @@ -266,8 +265,7 @@ public class TestResultSetLoaderMaps extends SubOperatorTest { .addRow(30, mapValue("barney")) .build(); - new RowSetComparison(expected).verifyAndClearAll(actual); - + RowSetUtilities.verify(expected, actual); rsLoader.close(); } @@ -318,7 +316,7 @@ public class TestResultSetLoaderMaps extends SubOperatorTest { .addRow(30, mapValue()) .build(); - new RowSetComparison(expected).verifyAndClearAll(actual); + RowSetUtilities.verify(expected, actual); // Now add another column to the map @@ -346,8 +344,7 @@ public class TestResultSetLoaderMaps extends SubOperatorTest { .addRow(50, mapValue("barney")) .build(); - new RowSetComparison(expected).verifyAndClearAll(actual); - + RowSetUtilities.verify(expected, actual); rsLoader.close(); } @@ -377,7 +374,7 @@ public class TestResultSetLoaderMaps extends SubOperatorTest { RowSetLoader rootWriter = rsLoader.writer(); rsLoader.startBatch(); - rootWriter.addRow(10, objArray("b1", objArray("c1"))); + rootWriter.addRow(10, mapValue("b1", mapValue("c1"))); // Validate first batch @@ -387,7 +384,7 @@ public class TestResultSetLoaderMaps extends SubOperatorTest { .addRow(10, mapValue("b1", mapValue("c1"))) .build(); - new RowSetComparison(expected).verifyAndClearAll(actual); + RowSetUtilities.verify(expected, actual); // Now add columns in the second batch. @@ -432,8 +429,7 @@ public class TestResultSetLoaderMaps extends SubOperatorTest { .addRow(40, mapValue("b4", mapValue("c4", "e4", "g4"), "d4", "e4")) .build(); - new RowSetComparison(expected).verifyAndClearAll(actual); - + RowSetUtilities.verify(expected, actual); rsLoader.close(); } @@ -471,7 +467,7 @@ public class TestResultSetLoaderMaps extends SubOperatorTest { // actual.print(); // expected.print(); - new RowSetComparison(expected).verifyAndClearAll(actual); + RowSetUtilities.verify(expected, actual); // Now add columns in the second batch. @@ -514,8 +510,7 @@ public class TestResultSetLoaderMaps extends SubOperatorTest { .addRow(40, mapValue("b4", mapValue("c4", "e4", "g4"), "d4", "e4")) .build(); - new RowSetComparison(expected).verifyAndClearAll(actual); - + RowSetUtilities.verify(expected, actual); rsLoader.close(); } @@ -561,7 +556,7 @@ public class TestResultSetLoaderMaps extends SubOperatorTest { .addRow(30, mapValue(intArray(), strArray("d3.1"))) .build(); - new RowSetComparison(expected).verifyAndClearAll(actual); + RowSetUtilities.verify(expected, actual); // Add another array after the first row in the second batch. http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java index 95aa2b7..7372e14 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java @@ -93,7 +93,7 @@ public class TestResultSetLoaderProjection extends SubOperatorTest { // All columns appear, including non-projected ones. - TupleMetadata actualSchema = rootWriter.schema(); + TupleMetadata actualSchema = rootWriter.tupleSchema(); assertEquals(4, actualSchema.size()); assertEquals("a", actualSchema.column(0).getName()); assertEquals("b", actualSchema.column(1).getName()); @@ -167,7 +167,7 @@ public class TestResultSetLoaderProjection extends SubOperatorTest { // Verify the projected columns - TupleMetadata actualSchema = rootWriter.schema(); + TupleMetadata actualSchema = rootWriter.tupleSchema(); ColumnMetadata m1Md = actualSchema.metadata("m1"); assertTrue(m1Md.isMap()); assertTrue(m1Md.isProjected()); http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java index c402466..30c20d7 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java @@ -41,8 +41,8 @@ import org.apache.drill.exec.vector.accessor.TupleWriter.UndefinedColumnExceptio import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.RowSet; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; -import org.apache.drill.test.rowSet.RowSetComparison; import org.apache.drill.test.rowSet.RowSetReader; +import org.apache.drill.test.rowSet.RowSetUtilities; import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.junit.Test; @@ -93,7 +93,7 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { // Can define schema before starting the first batch. RowSetLoader rootWriter = rsLoader.writer(); - TupleMetadata schema = rootWriter.schema(); + TupleMetadata schema = rootWriter.tupleSchema(); assertEquals(0, schema.size()); MaterializedField fieldA = SchemaBuilder.columnSchema("a", MinorType.INT, DataMode.REQUIRED); @@ -175,8 +175,7 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { .addRow(100, null) .addRow(200, 210) .build(); - new RowSetComparison(expected) - .verifyAndClearAll(result); + RowSetUtilities.verify(expected, result); // Between batches: batch-based operations fail @@ -229,8 +228,7 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { .addRow(300, 310) .addRow(400, 410) .build(); - new RowSetComparison(expected) - .verifyAndClearAll(result); + RowSetUtilities.verify(expected, result); // Next batch. Schema has changed. @@ -253,8 +251,7 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { .addRow(500, 510, 520) .addRow(600, 610, 620) .build(); - new RowSetComparison(expected) - .verifyAndClearAll(result); + RowSetUtilities.verify(expected, result); rsLoader.close(); @@ -312,7 +309,7 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { public void testCaseInsensitiveSchema() { ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator()); RowSetLoader rootWriter = rsLoader.writer(); - TupleMetadata schema = rootWriter.schema(); + TupleMetadata schema = rootWriter.tupleSchema(); assertEquals(0, rsLoader.schemaVersion()); // No columns defined in schema @@ -444,8 +441,7 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { .addRow("foo", "second", "", null, strArray()) .addRow("bar", "", "c.2", "d.2", strArray("e1", "e2", "e3")) .build(); - new RowSetComparison(expected) - .verifyAndClearAll(result); + RowSetUtilities.verify(expected, result); // Handy way to test that close works to abort an in-flight batch // and clean up. @@ -487,7 +483,7 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { .addRow(30, 300, "wilma") .build(); - new RowSetComparison(expected).verifyAndClearAll(actual); + RowSetUtilities.verify(expected, actual); rsLoader.close(); } http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java index 01c49ad..31109fe 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java @@ -69,7 +69,7 @@ public class RowSetUtilities { public static void setFromInt(RowSetWriter rowWriter, int index, int value) { ScalarWriter writer = rowWriter.scalar(index); - MaterializedField field = rowWriter.schema().column(index); + MaterializedField field = rowWriter.tupleSchema().column(index); writer.setObject(testDataFromInt(writer.valueType(), field.getType(), value)); } http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriterImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriterImpl.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriterImpl.java index aa7fb2f..8ba1f93 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriterImpl.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriterImpl.java @@ -19,6 +19,7 @@ package org.apache.drill.test.rowSet; import java.util.List; +import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; @@ -158,4 +159,12 @@ public class RowSetWriterImpl extends AbstractTupleWriter implements RowSetWrite public int lastWriteIndex() { return writerIndex.vectorIndex(); } + + @Override + public ColumnMetadata schema() { + // The top-level tuple (the data row) is not associated + // with a parent column. By contrast, a map tuple is + // associated with the column that defines the map. + return null; + } } http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/DummyWriterTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/DummyWriterTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/DummyWriterTest.java index c5277f8..d96878b 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/DummyWriterTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/DummyWriterTest.java @@ -17,16 +17,20 @@ */ package org.apache.drill.test.rowSet.test; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertEquals; import java.util.ArrayList; import java.util.List; import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.AbstractColumnMetadata; +import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.exec.vector.accessor.ValueType; import org.apache.drill.exec.vector.accessor.writer.AbstractObjectWriter; import org.apache.drill.exec.vector.accessor.writer.AbstractTupleWriter; import org.apache.drill.exec.vector.accessor.writer.ColumnWriterFactory; +import org.apache.drill.exec.vector.accessor.writer.MapWriter; import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.junit.Test; @@ -44,6 +48,9 @@ public class DummyWriterTest extends SubOperatorTest { List<AbstractObjectWriter> writers) { super(schema, writers); } + + @Override + public ColumnMetadata schema() { return null; } } /** @@ -73,7 +80,7 @@ public class DummyWriterTest extends SubOperatorTest { // At present, dummy writers report no type (because they don't have one.) - assertNull(rootWriter.scalar(0).valueType()); + assertEquals(ValueType.NULL, rootWriter.scalar(0).valueType()); // First column. Set int value. @@ -124,13 +131,20 @@ public class DummyWriterTest extends SubOperatorTest { .buildSchema(); List<AbstractObjectWriter> writers = new ArrayList<>(); + // Mark schema as non-projected + + ((AbstractColumnMetadata) schema.metadata("m1")).setProjected(false); + ((AbstractColumnMetadata) schema.metadata("m2")).setProjected(false); + + // Create the writers + { schema.metadata("m1").setProjected(false); TupleMetadata mapSchema = schema.metadata("m1").mapSchema(); List<AbstractObjectWriter> members = new ArrayList<>(); members.add(ColumnWriterFactory.buildColumnWriter(mapSchema.metadata("a"), null)); members.add(ColumnWriterFactory.buildColumnWriter(mapSchema.metadata("b"), null)); - writers.add(ColumnWriterFactory.buildMapWriter(schema.metadata("m1"), null, members)); + writers.add(MapWriter.buildMapWriter(schema.metadata("m1"), null, members)); } { @@ -138,7 +152,7 @@ public class DummyWriterTest extends SubOperatorTest { TupleMetadata mapSchema = schema.metadata("m2").mapSchema(); List<AbstractObjectWriter> members = new ArrayList<>(); members.add(ColumnWriterFactory.buildColumnWriter(mapSchema.metadata("c"), null)); - writers.add(ColumnWriterFactory.buildMapWriter(schema.metadata("m2"), null, members)); + writers.add(MapWriter.buildMapWriter(schema.metadata("m2"), null, members)); } AbstractTupleWriter rootWriter = new RootWriterFixture(schema, writers); http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/PerformanceTool.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/PerformanceTool.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/PerformanceTool.java index a2caeed..92ebdd5 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/PerformanceTool.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/PerformanceTool.java @@ -222,7 +222,8 @@ public class PerformanceTool { public void doTest() { try (NullableIntVector vector = new NullableIntVector(rowSchema.column(0), fixture.allocator());) { vector.allocateNew(ROW_COUNT); - NullableScalarWriter colWriter = new NullableScalarWriter( + ColumnMetadata colSchema = MetadataUtils.fromField(vector.getField()); + NullableScalarWriter colWriter = new NullableScalarWriter(colSchema, vector, new IntColumnWriter(vector.getValuesVector())); TestWriterIndex index = new TestWriterIndex(); colWriter.bindIndex(index); http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestIndirectReaders.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestIndirectReaders.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestIndirectReaders.java new file mode 100644 index 0000000..cdf2146 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestIndirectReaders.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.test.rowSet.test; + +import static org.junit.Assert.*; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.exec.record.selection.SelectionVector2; +import org.apache.drill.exec.vector.accessor.ArrayReader; +import org.apache.drill.exec.vector.accessor.ArrayWriter; +import org.apache.drill.exec.vector.accessor.ScalarReader; +import org.apache.drill.exec.vector.accessor.ScalarWriter; +import org.apache.drill.test.SubOperatorTest; +import org.apache.drill.test.rowSet.RowSetWriter; +import org.apache.drill.test.rowSet.RowSet.ExtendableRowSet; +import org.apache.drill.test.rowSet.RowSet.SingleRowSet; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; +import org.apache.drill.test.rowSet.RowSetComparison; +import org.apache.drill.test.rowSet.RowSetReader; +import org.junit.Test; + +/** + * Test reading with an indirection vector (sv2.) This form of + * indirection vector reorders values within a single batch. + * Since the indirection occurs only in the reader index, only + * light testing is done; all readers go through the same index class, + * so if the index works for one reader, it will for for all. + */ + +public class TestIndirectReaders extends SubOperatorTest { + + /** + * Simplest case: required reader, uses the index + * directly. + */ + + @Test + public void testRequired() { + TupleMetadata schema = new SchemaBuilder() + .add("a", MinorType.INT) + .buildSchema(); + ExtendableRowSet rowSet = fixture.rowSet(schema); + RowSetWriter writer = rowSet.writer(); + + // 10 rows with value 0 .. 9 + + for (int i = 0; i < 10; i++) { + writer.scalar(0).setInt(i); + writer.save(); + } + + SingleRowSet result = writer.done().toIndirect(); + + // Use the SV2 to reverse the row order. + + @SuppressWarnings("resource") + SelectionVector2 sv2 = result.getSv2(); + for (int i = 0; i < 10; i++) { + sv2.setIndex(i, 9 - i); + } + + // Values should be read back in the reverse order. + + RowSetReader reader = result.reader(); + for (int i = 9; i >= 0; i--) { + assertTrue(reader.next()); + assertEquals(i, reader.scalar(0).getInt()); + } + + // The row set comparison should read using the + // indirection. + + SingleRowSet expected = fixture.rowSetBuilder(schema) + .addRow(9) + .addRow(8) + .addRow(7) + .addRow(6) + .addRow(5) + .addRow(4) + .addRow(3) + .addRow(2) + .addRow(1) + .addRow(0) + .build(); + + new RowSetComparison(expected) + .verifyAndClearAll(result); + } + + /** + * More complex case with two levels of offset vector (one for the + * array, another for the Varchar values.) Only the top level goes + * through the indirection. + */ + + @Test + public void testArray() { + TupleMetadata schema = new SchemaBuilder() + .addArray("a", MinorType.VARCHAR) + .buildSchema(); + ExtendableRowSet rowSet = fixture.rowSet(schema); + RowSetWriter writer = rowSet.writer(); + ArrayWriter aWriter = writer.array(0); + ScalarWriter strWriter = aWriter.scalar(); + + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 5; j++) { + strWriter.setString("value" + i + "." + j); + } + writer.save(); + } + + SingleRowSet result = writer.done().toIndirect(); + + @SuppressWarnings("resource") + SelectionVector2 sv2 = result.getSv2(); + for (int i = 0; i < 10; i++) { + sv2.setIndex(i, 9 - i); + } + + RowSetReader reader = result.reader(); + ArrayReader aReader = reader.array(0); + ScalarReader strReader = aReader.scalar(); + + for (int i = 9; i >= 0; i--) { + assertTrue(reader.next()); + for (int j = 0; j < 5; j++) { + assertTrue(aReader.next()); + assertEquals("value" + i + "." + j, strReader.getString()); + } + } + + result.clear(); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestOffsetVectorWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestOffsetVectorWriter.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestOffsetVectorWriter.java index d2a99c4..af39205 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestOffsetVectorWriter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestOffsetVectorWriter.java @@ -29,7 +29,7 @@ import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.apache.drill.exec.vector.accessor.ScalarWriter.ColumnWriterListener; import org.apache.drill.exec.vector.accessor.ValueType; -import org.apache.drill.exec.vector.accessor.writer.OffsetVectorWriter; +import org.apache.drill.exec.vector.accessor.writer.OffsetVectorWriterImpl; import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.apache.drill.test.rowSet.test.TestFixedWidthWriter.TestIndex; @@ -81,7 +81,7 @@ public class TestOffsetVectorWriter extends SubOperatorTest { try (UInt4Vector vector = allocVector(1000)) { TestIndex index = new TestIndex(); - OffsetVectorWriter writer = makeWriter(vector, index); + OffsetVectorWriterImpl writer = makeWriter(vector, index); // Start write sets initial position to 0. @@ -120,7 +120,7 @@ public class TestOffsetVectorWriter extends SubOperatorTest { try (UInt4Vector vector = allocVector(1000)) { TestIndex index = new TestIndex(); - OffsetVectorWriter writer = makeWriter(vector, index); + OffsetVectorWriterImpl writer = makeWriter(vector, index); writer.startWrite(); // Write rows, rewriting every other row. @@ -162,7 +162,7 @@ public class TestOffsetVectorWriter extends SubOperatorTest { public void testFillEmpties() { try (UInt4Vector vector = allocVector(1000)) { TestIndex index = new TestIndex(); - OffsetVectorWriter writer = makeWriter(vector, index); + OffsetVectorWriterImpl writer = makeWriter(vector, index); writer.startWrite(); // Pretend to write offsets for values of width 10, but @@ -203,7 +203,7 @@ public class TestOffsetVectorWriter extends SubOperatorTest { public void testRollover() { try (UInt4Vector vector = allocVector(1000)) { TestIndex index = new TestIndex(); - OffsetVectorWriter writer = makeWriter(vector, index); + OffsetVectorWriterImpl writer = makeWriter(vector, index); writer.startWrite(); // Simulate doing an overflow of ten values. @@ -270,7 +270,7 @@ public class TestOffsetVectorWriter extends SubOperatorTest { public void testRolloverWithEmpties() { try (UInt4Vector vector = allocVector(1000)) { TestIndex index = new TestIndex(); - OffsetVectorWriter writer = makeWriter(vector, index); + OffsetVectorWriterImpl writer = makeWriter(vector, index); writer.startWrite(); // Simulate doing an overflow of 15 values, @@ -301,7 +301,7 @@ public class TestOffsetVectorWriter extends SubOperatorTest { // Verify the first "batch" results for (int i = 0; i < 11; i++) { - assertEquals(i * 10, vector.getAccessor().get(i)); + assertEquals("i = " + i, i * 10, vector.getAccessor().get(i)); } for (int i = 11; i < 16; i++) { assertEquals("i = " + i, 100, vector.getAccessor().get(i)); @@ -313,6 +313,10 @@ public class TestOffsetVectorWriter extends SubOperatorTest { vector.getMutator().set(i, 0xdeadbeef); } + // Simulate finishing the overflow row. + + index.index++; + // Post rollover, slot 0 should be initialized. // This is a rollover. This row must set the value // for the new row 0 (which was presumably set/filled @@ -344,10 +348,10 @@ public class TestOffsetVectorWriter extends SubOperatorTest { // Verify the results for (int i = 0; i < 6; i++) { - assertEquals(0, vector.getAccessor().get(i)); + assertEquals("Index + " + i, 0, vector.getAccessor().get(i)); } for (int i = 6; i < 11; i++) { - assertEquals((i - 5) * 10, vector.getAccessor().get(i)); + assertEquals("Index + " + i, (i - 5) * 10, vector.getAccessor().get(i)); } } } @@ -363,7 +367,7 @@ public class TestOffsetVectorWriter extends SubOperatorTest { public void testSizeLimit() { try (UInt4Vector vector = allocVector(1000)) { TestIndex index = new TestIndex(); - OffsetVectorWriter writer = makeWriter(vector, index); + OffsetVectorWriterImpl writer = makeWriter(vector, index); writer.bindListener(new ColumnWriterListener() { int totalAlloc = 4096; @@ -415,8 +419,8 @@ public class TestOffsetVectorWriter extends SubOperatorTest { return vector; } - private OffsetVectorWriter makeWriter(UInt4Vector vector, TestIndex index) { - OffsetVectorWriter writer = new OffsetVectorWriter(vector); + private OffsetVectorWriterImpl makeWriter(UInt4Vector vector, TestIndex index) { + OffsetVectorWriterImpl writer = new OffsetVectorWriterImpl(vector); writer.bindIndex(index); assertEquals(ValueType.INTEGER, writer.valueType()); http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/vector/src/main/codegen/templates/ColumnAccessors.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/codegen/templates/ColumnAccessors.java b/exec/vector/src/main/codegen/templates/ColumnAccessors.java index 4836099..d0a2ace 100644 --- a/exec/vector/src/main/codegen/templates/ColumnAccessors.java +++ b/exec/vector/src/main/codegen/templates/ColumnAccessors.java @@ -268,69 +268,58 @@ public class ColumnAccessors { </#if> <@getType drillType label /> - <#if accessorType == "byte[]"> - <#assign args = ", int len"> - <#else> - <#assign args = ""> - </#if> - <#if javaType == "char"> - <#assign putType = "short" /> - <#assign doCast = true /> - <#else> - <#assign putType = javaType /> - <#assign doCast = (cast == "set") /> - </#if> <#if ! varWidth> </#if> @Override - public final void set${label}(final ${accessorType} value${args}) { + public final void set${label}(final ${accessorType} value${putArgs}) { <#-- Must compute the write offset first; can't be inline because the writeOffset() function has a side effect of possibly changing the buffer address (bufAddr). --> - <#if varWidth> - final int offset = writeIndex(len); - <#else> - final int writeIndex = writeIndex(); - <#assign putAddr = "writeIndex * VALUE_WIDTH"> + <#if ! varWidth> + final int writeOffset = prepareWrite(); + <#assign putOffset = "writeOffset * VALUE_WIDTH"> </#if> <#if varWidth> + final int offset = prepareWrite(len); drillBuf.setBytes(offset, value, 0, len); offsetsWriter.setNextOffset(offset + len); <#elseif drillType == "Decimal9"> - drillBuf.setInt(${putAddr}, + drillBuf.setInt(${putOffset}, DecimalUtility.getDecimal9FromBigDecimal(value, - type.getScale(), type.getPrecision())); + type.getScale(), type.getPrecision())); <#elseif drillType == "Decimal18"> - drillBuf.setLong(${putAddr}, + drillBuf.setLong(${putOffset}, DecimalUtility.getDecimal18FromBigDecimal(value, - type.getScale(), type.getPrecision())); + type.getScale(), type.getPrecision())); <#elseif drillType == "Decimal38Sparse"> <#-- Hard to optimize this case. Just use the available tools. --> - DecimalUtility.getSparseFromBigDecimal(value, vector.getBuffer(), writeIndex * VALUE_WIDTH, - type.getScale(), type.getPrecision(), 6); + DecimalUtility.getSparseFromBigDecimal(value, drillBuf, + ${putOffset}, + type.getScale(), type.getPrecision(), 6); <#elseif drillType == "Decimal28Sparse"> <#-- Hard to optimize this case. Just use the available tools. --> - DecimalUtility.getSparseFromBigDecimal(value, vector.getBuffer(), writeIndex * VALUE_WIDTH, - type.getScale(), type.getPrecision(), 5); + DecimalUtility.getSparseFromBigDecimal(value, drillBuf, + ${putOffset}, + type.getScale(), type.getPrecision(), 5); <#elseif drillType == "IntervalYear"> - drillBuf.setInt(${putAddr}, - value.getYears() * 12 + value.getMonths()); + drillBuf.setInt(${putOffset}, + value.getYears() * 12 + value.getMonths()); <#elseif drillType == "IntervalDay"> - final int offset = ${putAddr}; - drillBuf.setInt(offset, value.getDays()); - drillBuf.setInt(offset + 4, DateUtilities.periodToMillis(value)); + final int offset = ${putOffset}; + drillBuf.setInt(offset, value.getDays()); + drillBuf.setInt(offset + ${minor.millisecondsOffset}, DateUtilities.periodToMillis(value)); <#elseif drillType == "Interval"> - final int offset = ${putAddr}; - drillBuf.setInt(offset, value.getYears() * 12 + value.getMonths()); - drillBuf.setInt(offset + 4, value.getDays()); - drillBuf.setInt(offset + 8, DateUtilities.periodToMillis(value)); + final int offset = ${putOffset}; + drillBuf.setInt(offset, DateUtilities.periodToMonths(value)); + drillBuf.setInt(offset + ${minor.daysOffset}, value.getDays()); + drillBuf.setInt(offset + ${minor.millisecondsOffset}, DateUtilities.periodToMillis(value)); <#elseif drillType == "Float4"> - drillBuf.setInt(${putAddr}, Float.floatToRawIntBits((float) value)); + drillBuf.setInt(${putOffset}, Float.floatToRawIntBits((float) value)); <#elseif drillType == "Float8"> - drillBuf.setLong(${putAddr}, Double.doubleToRawLongBits(value)); + drillBuf.setLong(${putOffset}, Double.doubleToRawLongBits(value)); <#else> - drillBuf.set${putType?cap_first}(${putAddr}, <#if doCast>(${putType}) </#if>value); + drillBuf.set${putType?cap_first}(${putOffset}, <#if doCast>(${putType}) </#if>value); </#if> vectorIndex.nextElement(); } http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ProjectionType.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ProjectionType.java b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ProjectionType.java index 4972605..12f0f11 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ProjectionType.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ProjectionType.java @@ -24,4 +24,4 @@ public enum ProjectionType { TUPLE, // x.y ARRAY, // x[0] TUPLE_ARRAY // x[0].y -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ArrayWriter.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ArrayWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ArrayWriter.java index 49a1e77..c15791a 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ArrayWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ArrayWriter.java @@ -22,10 +22,42 @@ package org.apache.drill.exec.vector.accessor; * each call to a <tt>setFoo()</tt> method writes a value and advances the array * index. * <p> - * {@see ArrayReader} + * The array writer represents a Drill repeated type, including repeated maps. + * The array writer also represents the Drill list and repeated list types as + * follows: + * <ul> + * <li>A repeated scalar type is presented as an array writer with scalar + * entries. As a convenience, writing to the scalar automatically advances + * the current array write position, since exactly one item can be written + * per array entry.</li> + * <li>A repeated map type is presented as an array writer with tuple + * entries. The client must advance the array write position explicitly since + * a tuple can have any number of entries and the array writer cannot determine + * when a value is complete.</li> + * <li>A list type is presented as an array of variant entries. The client + * must explicitly advance the array position.</li> + * <li>A repeated list type is presented as an array of arrays of variants. + * The client advances the array position of both lists.</li> + * <li>Lists of repeated lists have three levels of arrays, repeated lists + * of repeated lists have four levels of arrays, and so on.</li> + * </ul> + * <p> + * Although the list vector supports a union of any Drill type, the only sane + * combinations are: + * <ul> + * <li>One of a (single or repeated) (map or list), or</li> + * <li>One or more scalar type.</li> + * </ul> + * + * If a particular array has only one type (single/repeated map/list), then, + * for convenience, the caller can directly request a writer of that type + * without having to first retrieve the variant (although the indirect + * route is, of course, available.) + * + * @see {@link ArrayReader} */ -public interface ArrayWriter { +public interface ArrayWriter extends ColumnWriter { /** * Number of elements written thus far to the array. @@ -35,20 +67,22 @@ public interface ArrayWriter { int size(); /** - * The object type of the list entry. All entries have the same - * type. - * @return the object type of each entry - */ - - ObjectWriter entry(); - - /** * Return a generic object writer for the array entry. * * @return generic object reader */ ObjectType entryType(); + + void setNull(boolean isNull); + + /** + * The object type of the list entry. All entries have the same + * type. + * @return the object type of each entry + */ + + ObjectWriter entry(); ScalarWriter scalar(); TupleWriter tuple(); ArrayWriter array(); @@ -60,23 +94,4 @@ public interface ArrayWriter { */ void save(); - - /** - * Write the values of an array from a list of arguments. - * @param values values for each array element - * @throws VectorOverflowException - */ - void set(Object ...values); - - /** - * Write the array given an array of values. The type of array must match - * the type of element in the array. That is, if the value is an <tt>int</tt>, - * provide an <tt>int[]</tt> array. - * - * @param array array of values to write - * @throws VectorOverflowException - */ - - void setObject(Object array); -// void setList(List<? extends Object> list); } http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnWriter.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnWriter.java new file mode 100644 index 0000000..5d1e79f --- /dev/null +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnWriter.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.vector.accessor; + +import org.apache.drill.exec.record.metadata.ColumnMetadata; +import org.apache.drill.exec.vector.accessor.ScalarWriter.ColumnWriterListener; +import org.apache.drill.exec.vector.accessor.TupleWriter.TupleWriterListener; + +/** + * Generic information about a column writer including: + * <ul> + * <li>Metadata</li> + * <li>Write position information about a writer needed by a vector overflow + * implementation. Hides the details of implementation and the writer class + * hierarchy, exposing just the required write position information.</li> + * <li>Generic methods for writing to the object, primarily used for + * testing.</li> + */ + +public interface ColumnWriter extends WriterPosition { + + interface TupleListenable { + + /** + * Bind a listener to the underlying map or map array column. Not valid if the + * underlying writer is a scalar or scalar array. + * + * @param listener + * the tuple listener to bind + */ + + void bindListener(TupleWriterListener listener); + } + + interface ScalarListenable { + /** + * Bind a listener to the underlying scalar column, or array of scalar + * columns. Not valid if the underlying writer is a map or array of maps. + * + * @param listener + * the column listener to bind + */ + + void bindListener(ColumnWriterListener listener); + } + + /** + * Return the object (structure) type of this writer. + * + * @return type indicating if this is a scalar, tuple or array + */ + + ObjectType type(); + + /** + * Whether this writer allows nulls. This is not as simple as checking + * for the {@link DataMode#OPTIONAL} type in the schema. List entries + * are nullable, if they are primitive, but not if they are maps or lists. + * Unions are nullable, regardless of cardinality. + * + * @return true if a call to {@link #setNull()} is supported, false + * if not + */ + + boolean nullable(); + + /** + * Returns the schema of the column associated with this writer. + * + * @return schema for this writer's column + */ + + ColumnMetadata schema(); + + /** + * Set the current value to null. Support depends on the underlying + * implementation: only nullable types support this operation. + * + * throws IllegalStateException if called on a non-nullable value. + */ + + void setNull(); + + /** + * Generic technique to write data as a generic Java object. The + * type of the object must match the target writer. + * Primarily for testing. + * <ul> + * <li>Scalar: The type of the Java object must match the type of + * the target vector. <tt>String</tt> or <tt>byte[]</tt> can be + * used for Varchar vectors.</li> + * <li>Array: Write the array given an array of values. The object + * must be a Java array. The type of the array must match the type of + * element in the repeated vector. That is, if the vector is + * a <tt>Repeated Int</tt>, provide an <tt>int[]</tt> array.</tt></li> + * <li>Tuple (map or row): The Java object must be an array of objects + * in which the members of the array have a 1:1 correspondence with the + * members of the tuple in the order defined by the writer metadata. + * That is, if the map is (Int, Varchar), provide a <tt>Object[]</tt> + * array like this: <tt>{10, "fred"}</tt>.</li> + * <li>Union: Uses the Java object type to determine the type of the + * backing vector. Creates a vector + * of the required type if needed.</li> + * + * @param value value to write to the vector. The Java type of the + * object indicates the Drill storage type + * @throws IllegalArgumentException if the type of the Java object + * cannot be mapped to the type of the underlying vector or + * vector structure + */ + + void setObject(Object value); +} http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ObjectWriter.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ObjectWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ObjectWriter.java index 113778f..81d4bac 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ObjectWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ObjectWriter.java @@ -17,10 +17,6 @@ */ package org.apache.drill.exec.vector.accessor; -import org.apache.drill.exec.record.metadata.ColumnMetadata; -import org.apache.drill.exec.vector.accessor.ScalarWriter.ColumnWriterListener; -import org.apache.drill.exec.vector.accessor.TupleWriter.TupleWriterListener; - /** * Represents a column within a tuple. A column can be an array, a scalar or a * tuple. Each has an associated column metadata (schema) and a writer. The @@ -44,58 +40,11 @@ import org.apache.drill.exec.vector.accessor.TupleWriter.TupleWriterListener; * {@see ObjectReader} */ -public interface ObjectWriter { - - /** - * Returns the schema of the column associated with this writer. - * - * @return schema for this writer's column - */ - - ColumnMetadata schema(); - - /** - * Bind a listener to the underlying scalar column, or array of scalar - * columns. Not valid if the underlying writer is a map or array of maps. - * - * @param listener - * the column listener to bind - */ - - void bindListener(ColumnWriterListener listener); - - /** - * Bind a listener to the underlying map or map array column. Not valid if the - * underlying writer is a scalar or scalar array. - * - * @param listener - * the tuple listener to bind - */ - - void bindListener(TupleWriterListener listener); - - /** - * Return the object (structure) type of this writer. - * - * @return type indicating if this is a scalar, tuple or array - */ - - ObjectType type(); +public interface ObjectWriter extends ColumnWriter { ScalarWriter scalar(); TupleWriter tuple(); ArrayWriter array(); - - /** - * For debugging, set the object to the proper form of Java object as defined - * by the underlying writer type. - * - * @param value - * Java object value to write - * @throws VectorOverflowException - */ - - void set(Object value); } http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarWriter.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarWriter.java index 776dc9c..87c7988 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarWriter.java @@ -21,6 +21,8 @@ import java.math.BigDecimal; import org.joda.time.Period; +import org.apache.drill.exec.vector.accessor.ColumnWriter.ScalarListenable; + /** * Represents a scalar value: a required column, a nullable column, * or one element within an array of scalars. @@ -42,7 +44,7 @@ import org.joda.time.Period; * {@see ScalarElementReader} */ -public interface ScalarWriter { +public interface ScalarWriter extends ColumnWriter, ScalarListenable { /** * Listener (callback) for vector overflow events. To be optionally @@ -78,8 +80,6 @@ public interface ScalarWriter { boolean canExpand(ScalarWriter writer, int delta); } - void bindListener(ColumnWriterListener listener); - /** * Describe the type of the value. This is a compression of the * value vector type: it describes which method will return the @@ -89,7 +89,6 @@ public interface ScalarWriter { */ ValueType valueType(); - void setNull(); void setInt(int value); void setLong(long value); void setDouble(double value); @@ -97,6 +96,4 @@ public interface ScalarWriter { void setBytes(byte[] value, int len); void setDecimal(BigDecimal value); void setPeriod(Period value); - - void setObject(Object value); } http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleWriter.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleWriter.java index 0a52283..331df2a 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleWriter.java @@ -19,7 +19,9 @@ package org.apache.drill.exec.vector.accessor; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.metadata.ColumnMetadata; +import org.apache.drill.exec.record.metadata.ProjectionType; import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.exec.vector.accessor.ColumnWriter.TupleListenable; /** * Writer for a tuple. A tuple is composed of columns with a fixed order and @@ -50,7 +52,7 @@ import org.apache.drill.exec.record.metadata.TupleMetadata; * @see {@link SingleMapWriter}, the class which this class replaces */ -public interface TupleWriter { +public interface TupleWriter extends ColumnWriter, TupleListenable { /** * Listener (callback) to handle requests to add a new column to a tuple (row @@ -59,10 +61,13 @@ public interface TupleWriter { * throws an exception. */ - public interface TupleWriterListener { + interface TupleWriterListener { + ObjectWriter addColumn(TupleWriter tuple, ColumnMetadata column); ObjectWriter addColumn(TupleWriter tuple, MaterializedField field); + + ProjectionType projectionType(String columnName); } /** @@ -75,13 +80,26 @@ public interface TupleWriter { */ @SuppressWarnings("serial") - public static class UndefinedColumnException extends RuntimeException { + class UndefinedColumnException extends RuntimeException { public UndefinedColumnException(String colName) { super("Undefined column: " + colName); } } - void bindListener(TupleWriterListener listener); + + /** + * Allows a client to "sniff" the projection set to determine if a + * field is projected. Some clients can omit steps if they know that + * a field is not needed. Others will simply create the column, allowing + * the implementation to create a dummy writer if the column is not + * projected. + * + * @param columnName name of an existing or new column + * @return whether the column is projected, and, if so, the implied + * type of the projected column + */ + + ProjectionType projectionType(String columnName); /** * Add a column to the tuple (row or map) that backs this writer. Support for @@ -100,7 +118,7 @@ public interface TupleWriter { int addColumn(MaterializedField schema); - TupleMetadata schema(); + TupleMetadata tupleSchema(); int size(); @@ -142,28 +160,4 @@ public interface TupleWriter { */ void set(int colIndex, Object value); - - /** - * Write a row or map of values, given by Java objects. Object type must match - * expected column type. - * <p> - * Note that a single-column tuple is ambiguous if that column is an array. To - * avoid ambiguity, use <tt>set(0, value)</tt> in this case. - * - * @param values - * variable-length argument list of column values - * @return true if the row was written, false if any column caused vector - * overflow. - */ - - void setTuple(Object... values); - - /** - * Set the tuple from an array of objects. Primarily for use in test tools. - * - * @param value - * the object to set, which must be a generic <tt>Object</tt> array - */ - - void setObject(Object value); } http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/WriterPosition.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/WriterPosition.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/WriterPosition.java new file mode 100644 index 0000000..609545a --- /dev/null +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/WriterPosition.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.vector.accessor; + +/** + * Position information about a writer used during vector overflow. + */ + +public interface WriterPosition { + + /** + * Position within the vector of the first value for the current row. + * Note that this is always the first value for the row, even for a + * writer deeply nested within a hierarchy of arrays. (The first + * position for the current array is not exposed in this API.) + * + * @return the vector offset of the first value for the current + * row + */ + + int rowStartIndex(); + + /** + * Return the last write position in the vector. This may be the + * same as the writer index position (if the vector was written at + * that point), or an earlier point. In either case, this value + * points to the last valid value in the vector. + * + * @return index of the last valid value in the vector + */ + + int lastWriteIndex(); + + /** + * Current write index for the writer. This is the global + * array location for arrays, same as the row index for top-level + * columns. + * + * @return current write index + */ + + int writeIndex(); +} http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractArrayWriter.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractArrayWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractArrayWriter.java index 58cda57..2a2e3e1 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractArrayWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractArrayWriter.java @@ -24,9 +24,7 @@ import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; import org.apache.drill.exec.vector.accessor.ObjectType; import org.apache.drill.exec.vector.accessor.ObjectWriter; import org.apache.drill.exec.vector.accessor.ScalarWriter; -import org.apache.drill.exec.vector.accessor.ScalarWriter.ColumnWriterListener; import org.apache.drill.exec.vector.accessor.TupleWriter; -import org.apache.drill.exec.vector.accessor.TupleWriter.TupleWriterListener; import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; /** @@ -98,36 +96,17 @@ public abstract class AbstractArrayWriter implements ArrayWriter, WriterEvents { private AbstractArrayWriter arrayWriter; - public ArrayObjectWriter(ColumnMetadata schema, AbstractArrayWriter arrayWriter) { - super(schema); + public ArrayObjectWriter(AbstractArrayWriter arrayWriter) { this.arrayWriter = arrayWriter; } @Override - public ObjectType type() { return ObjectType.ARRAY; } - - @Override - public void set(Object value) { - arrayWriter.setObject(value); - } - - @Override public ArrayWriter array() { return arrayWriter; } @Override public WriterEvents events() { return arrayWriter; } @Override - public void bindListener(ColumnWriterListener listener) { - arrayWriter.bindListener(listener); - } - - @Override - public void bindListener(TupleWriterListener listener) { - arrayWriter.bindListener(listener); - } - - @Override public void dump(HierarchicalFormatter format) { format .startObject(this) @@ -137,62 +116,57 @@ public abstract class AbstractArrayWriter implements ArrayWriter, WriterEvents { } } - public static abstract class BaseArrayWriter extends AbstractArrayWriter { - - /** - * Index into the vector of elements for a repeated vector. - * Keeps track of the current offset in terms of value positions. - * Forwards overflow events to the base index. - */ + /** + * Index into the vector of elements for a repeated vector. + * Keeps track of the current offset in terms of value positions. + * Forwards overflow events to the base index. + */ - public class ArrayElementWriterIndex implements ColumnWriterIndex { + public class ArrayElementWriterIndex implements ColumnWriterIndex { - private int elementIndex; + private int elementIndex; - public void reset() { elementIndex = 0; } + public void reset() { elementIndex = 0; } - @Override - public int vectorIndex() { return elementIndex + offsetsWriter.nextOffset(); } + @Override + public int vectorIndex() { return elementIndex + offsetsWriter.nextOffset(); } - @Override - public int rowStartIndex() { return offsetsWriter.rowStartOffset(); } + @Override + public int rowStartIndex() { return offsetsWriter.rowStartOffset(); } - public int arraySize() { return elementIndex; } + public int arraySize() { return elementIndex; } - @Override - public void nextElement() { } + @Override + public void nextElement() { } - public final void next() { elementIndex++; } + public void next() { elementIndex++; } - public int valueStartOffset() { return offsetsWriter.nextOffset(); } + public int valueStartOffset() { return offsetsWriter.nextOffset(); } - @Override - public void rollover() { } + @Override + public void rollover() { } - @Override - public ColumnWriterIndex outerIndex() { - return outerIndex; - } + @Override + public ColumnWriterIndex outerIndex() { + return outerIndex; + } - @Override - public String toString() { - return new StringBuilder() - .append("[") - .append(getClass().getSimpleName()) - .append(" elementIndex = ") - .append(elementIndex) - .append("]") - .toString(); - } + @Override + public String toString() { + return new StringBuilder() + .append("[") + .append(getClass().getSimpleName()) + .append(" elementIndex = ") + .append(elementIndex) + .append("]") + .toString(); } + } - private final OffsetVectorWriter offsetsWriter; - private ColumnWriterIndex outerIndex; - protected ArrayElementWriterIndex elementIndex; + public static abstract class BaseArrayWriter extends AbstractArrayWriter { - public BaseArrayWriter(UInt4Vector offsetVector, AbstractObjectWriter elementObjWriter) { - super(elementObjWriter); - offsetsWriter = new OffsetVectorWriter(offsetVector); + public BaseArrayWriter(ColumnMetadata schema, UInt4Vector offsetVector, AbstractObjectWriter elementObjWriter) { + super(schema, elementObjWriter, new OffsetVectorWriterImpl(offsetVector)); } @Override @@ -204,12 +178,6 @@ public abstract class AbstractArrayWriter implements ArrayWriter, WriterEvents { } @Override - public ColumnWriterIndex writerIndex() { return outerIndex; } - - @Override - public int size() { return elementIndex.arraySize(); } - - @Override public void startWrite() { elementIndex.reset(); offsetsWriter.startWrite(); @@ -248,12 +216,6 @@ public abstract class AbstractArrayWriter implements ArrayWriter, WriterEvents { } @Override - public void endWrite() { - offsetsWriter.endWrite(); - elementObjWriter.events().endWrite(); - } - - @Override public void preRollover() { elementObjWriter.events().preRollover(); offsetsWriter.preRollover(); @@ -271,28 +233,13 @@ public abstract class AbstractArrayWriter implements ArrayWriter, WriterEvents { } @Override - public int lastWriteIndex() { return outerIndex.vectorIndex(); } - - /** - * Return the writer for the offset vector for this array. Primarily used - * to handle overflow; other clients should not attempt to muck about with - * the offset vector directly. - * - * @return the writer for the offset vector associated with this array - */ - - @Override - public OffsetVectorWriter offsetWriter() { return offsetsWriter; } - - @Override - public void bindListener(ColumnWriterListener listener) { - elementObjWriter.bindListener(listener); + public void endWrite() { + offsetsWriter.endWrite(); + elementObjWriter.events().endWrite(); } @Override - public void bindListener(TupleWriterListener listener) { - elementObjWriter.bindListener(listener); - } + public int lastWriteIndex() { return outerIndex.vectorIndex(); } @Override public void dump(HierarchicalFormatter format) { @@ -305,18 +252,30 @@ public abstract class AbstractArrayWriter implements ArrayWriter, WriterEvents { } } - protected final AbstractObjectWriter elementObjWriter; + protected final ColumnMetadata schema; + protected AbstractObjectWriter elementObjWriter; + protected final OffsetVectorWriter offsetsWriter; + protected ColumnWriterIndex outerIndex; + protected ArrayElementWriterIndex elementIndex; - public AbstractArrayWriter(AbstractObjectWriter elementObjWriter) { + public AbstractArrayWriter(ColumnMetadata schema, AbstractObjectWriter elementObjWriter, OffsetVectorWriter offsetVectorWriter) { + this.schema = schema; this.elementObjWriter = elementObjWriter; + this.offsetsWriter = offsetVectorWriter; } @Override + public ObjectType type() { return ObjectType.ARRAY; } + + @Override public ObjectType entryType() { return elementObjWriter.type(); } @Override + public ColumnMetadata schema() { return schema; } + + @Override public ObjectWriter entry() { return elementObjWriter; } @Override @@ -334,9 +293,48 @@ public abstract class AbstractArrayWriter implements ArrayWriter, WriterEvents { return elementObjWriter.array(); } - public abstract void bindListener(ColumnWriterListener listener); - public abstract void bindListener(TupleWriterListener listener); - public abstract OffsetVectorWriter offsetWriter(); + @Override + public int size() { return elementIndex.arraySize(); } + + @Override + public boolean nullable() { return false; } + + @Override + public void setNull() { + throw new IllegalStateException("Not nullable"); + } + + @Override + public int rowStartIndex() { + return outerIndex.rowStartIndex(); + } + + @Override + public int lastWriteIndex() { + return offsetsWriter.lastWriteIndex(); + } + + @Override + public int writeIndex() { + return outerIndex.vectorIndex(); + } + + @Override + public void setNull(boolean isNull) { + if (isNull == true) { + throw new UnsupportedOperationException(); + } + } + + /** + * Return the writer for the offset vector for this array. Primarily used + * to handle overflow; other clients should not attempt to muck about with + * the offset vector directly. + * + * @return the writer for the offset vector associated with this array + */ + + public OffsetVectorWriter offsetWriter() { return offsetsWriter; } public void dump(HierarchicalFormatter format) { format http://git-wip-us.apache.org/repos/asf/drill/blob/dbff1646/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractFixedWidthWriter.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractFixedWidthWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractFixedWidthWriter.java index 1107216..921cb00 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractFixedWidthWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractFixedWidthWriter.java @@ -46,14 +46,14 @@ public abstract class AbstractFixedWidthWriter extends BaseScalarWriter { * The current vector buffer, and buffer address, will change in * this method when a vector grows or overflows. So, don't use this * method in inline calls of the form<br><code> - * vector.getBuffer().doSomething(writeIndex());</code></br> + * vector.getBuffer().doSomething(prepareWrite());</code></br> * The buffer obtained by <tt>getBuffer()</tt> can be different than - * the current buffer after <tt>writeIndex()</tt>. + * the current buffer after <tt>prepareWrite()</tt>. * * @return the index at which to write the current value */ - protected final int writeIndex() { + protected final int prepareWrite() { // "Fast path" for the normal case of no fills, no overflow. // This is the only bounds check we want to do for the entire @@ -191,6 +191,18 @@ public abstract class AbstractFixedWidthWriter extends BaseScalarWriter { @Override public int lastWriteIndex() { return lastWriteIndex; } + /** + * For internal use only to update the write position on those + * very rare occasions in which the vector is written to outside + * of this writer framework. Not to be called by application code! + * + * @param index new last write index + */ + + public void setLastWriteIndex(int index) { + lastWriteIndex = index; + } + @Override public void skipNulls() {
