http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java index 352ab34..c402466 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java @@ -433,7 +433,7 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { assertEquals(5, schema.size()); assertEquals(4, schema.index("e")); assertEquals(4, schema.index("E")); - rootWriter.array(4).set("e1", "e2", "e3"); + rootWriter.array(4).setObject(strArray("e1", "e2", "e3")); rootWriter.save(); // Verify. No reason to expect problems, but might as well check.
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java index cc2a83e..07100ed 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java @@ -29,7 +29,6 @@ import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.ArrayReader; import org.apache.drill.exec.vector.accessor.ArrayWriter; -import org.apache.drill.exec.vector.accessor.ScalarElementReader; import org.apache.drill.exec.vector.accessor.ScalarReader; import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.apache.drill.exec.vector.accessor.TupleReader; @@ -256,7 +255,8 @@ public class TestResultSetLoaderTorture extends SubOperatorTest { ScalarReader n2Reader; ScalarReader s2Reader; ScalarReader n3Reader; - ScalarElementReader s3Reader; + ArrayReader s3Array; + ScalarReader s3Reader; ReadState readState; public BatchReader(TestSetup setup, RowSetReader reader, ReadState readState) { @@ -272,7 +272,8 @@ public class TestResultSetLoaderTorture extends SubOperatorTest { s2Reader = m2Reader.scalar("s2"); TupleReader m3Reader = m2Reader.tuple("m3"); n3Reader = m3Reader.scalar("n3"); - s3Reader = m3Reader.array("s3").elements(); + s3Array = m3Reader.array("s3"); + s3Reader = s3Array.scalar(); } public void verify() { @@ -312,7 +313,7 @@ public class TestResultSetLoaderTorture extends SubOperatorTest { private void verifyM2Array() { for (int i = 0; i < setup.m2Count; i++) { - a2Reader.setPosn(i); + assert(a2Reader.next()); // n2: usual int @@ -321,7 +322,7 @@ public class TestResultSetLoaderTorture extends SubOperatorTest { if (readState.innerCount % setup.s2Cycle == 0) { // Skipped values should be null assertTrue( - String.format("Row %d, entry %d", rootReader.rowIndex(), i), + String.format("Row %d, entry %d", rootReader.offset(), i), s2Reader.isNull()); } else if (readState.innerCount % setup.s2Cycle % setup.nullCycle == 0) { assertTrue(s2Reader.isNull()); @@ -338,10 +339,11 @@ public class TestResultSetLoaderTorture extends SubOperatorTest { // s3: a repeated VarChar if (readState.innerCount % setup.s3Cycle == 0) { - assertEquals(0, s3Reader.size()); + assertEquals(0, s3Array.size()); } else { for (int j = 0; j < setup.s3Count; j++) { - assertEquals(setup.s3Value + (readState.innerCount * setup.s3Count + j), s3Reader.getString(j)); + assertTrue(s3Array.next()); + assertEquals(setup.s3Value + (readState.innerCount * setup.s3Count + j), s3Reader.getString()); } } readState.innerCount++; http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestVectorContainer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestVectorContainer.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestVectorContainer.java index a511a0a..0882a1c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestVectorContainer.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestVectorContainer.java @@ -28,6 +28,7 @@ import org.apache.drill.test.rowSet.RowSet; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.apache.drill.test.rowSet.RowSetComparison; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -72,7 +73,7 @@ public class TestVectorContainer extends DrillTest { .addRow(30, "wilma") .build(); - // Simulated "implicit" coumns: row number and file name + // Simulated "implicit" columns: row number and file name BatchSchema rightSchema = new SchemaBuilder() .add("x", MinorType.SMALLINT) http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/DirectRowSet.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/DirectRowSet.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/DirectRowSet.java index 9262706..0e63c0e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/DirectRowSet.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/DirectRowSet.java @@ -20,17 +20,17 @@ package org.apache.drill.test.rowSet; import com.google.common.collect.Sets; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.physical.rowSet.model.MetadataProvider.MetadataRetrieval; -import org.apache.drill.exec.physical.rowSet.model.ReaderIndex; -import org.apache.drill.exec.physical.rowSet.model.SchemaInference; import org.apache.drill.exec.physical.rowSet.model.single.BaseWriterBuilder; import org.apache.drill.exec.physical.rowSet.model.single.BuildVectorsFromMetadata; +import org.apache.drill.exec.physical.rowSet.model.single.DirectRowIndex; +import org.apache.drill.exec.physical.rowSet.model.single.SingleSchemaInference; import org.apache.drill.exec.physical.rowSet.model.single.VectorAllocator; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; -import org.apache.drill.exec.record.VectorAccessible; -import org.apache.drill.exec.record.VectorContainer; import org.apache.drill.exec.record.metadata.MetadataUtils; import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.exec.record.VectorAccessible; +import org.apache.drill.exec.record.VectorContainer; import org.apache.drill.exec.record.selection.SelectionVector2; import org.apache.drill.test.rowSet.RowSet.ExtendableRowSet; import org.apache.drill.test.rowSet.RowSetWriterImpl.WriterIndexImpl; @@ -44,26 +44,6 @@ import java.util.Set; public class DirectRowSet extends AbstractSingleRowSet implements ExtendableRowSet { - /** - * Reader index that points directly to each row in the row set. - * This index starts with pointing to the -1st row, so that the - * reader can require a <tt>next()</tt> for every row, including - * the first. (This is the JDBC RecordSet convention.) - */ - - private static class DirectRowIndex extends ReaderIndex { - - public DirectRowIndex(int rowCount) { - super(rowCount); - } - - @Override - public int vectorIndex() { return rowIndex; } - - @Override - public int batchIndex() { return 0; } - } - public static class RowSetWriterBuilder extends BaseWriterBuilder { public RowSetWriter buildWriter(DirectRowSet rowSet) { @@ -94,7 +74,7 @@ public class DirectRowSet extends AbstractSingleRowSet implements ExtendableRowS } public static DirectRowSet fromContainer(VectorContainer container) { - return new DirectRowSet(container, new SchemaInference().infer(container)); + return new DirectRowSet(container, new SingleSchemaInference().infer(container)); } public static DirectRowSet fromVectorAccessible(BufferAllocator allocator, VectorAccessible va) { @@ -151,7 +131,6 @@ public class DirectRowSet extends AbstractSingleRowSet implements ExtendableRowS return new IndirectRowSet(this, skipIndices); } - @Override public SelectionVector2 getSv2() { return null; } } http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/HyperRowSetImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/HyperRowSetImpl.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/HyperRowSetImpl.java index 70b8a20..542421e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/HyperRowSetImpl.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/HyperRowSetImpl.java @@ -17,11 +17,16 @@ */ package org.apache.drill.test.rowSet; -import org.apache.drill.exec.physical.rowSet.model.MetadataProvider.MetadataRetrieval; -import org.apache.drill.exec.physical.rowSet.model.SchemaInference; +import java.util.ArrayList; +import java.util.List; + +import org.apache.drill.exec.exception.SchemaChangeException; +import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.physical.rowSet.model.hyper.BaseReaderBuilder; +import org.apache.drill.exec.physical.rowSet.model.hyper.HyperSchemaInference; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.exec.record.ExpandableHyperContainer; import org.apache.drill.exec.record.VectorContainer; import org.apache.drill.exec.record.selection.SelectionVector4; import org.apache.drill.test.rowSet.RowSet.HyperRowSet; @@ -45,8 +50,67 @@ public class HyperRowSetImpl extends AbstractRowSet implements HyperRowSet { TupleMetadata schema = rowSet.schema(); HyperRowIndex rowIndex = new HyperRowIndex(sv4); return new RowSetReaderImpl(schema, rowIndex, - buildContainerChildren(rowSet.container(), - new MetadataRetrieval(schema))); + buildContainerChildren(rowSet.container(), schema)); + } + } + + public static class HyperRowSetBuilderImpl implements HyperRowSetBuilder { + + private final BufferAllocator allocator; + private final List<VectorContainer> batches = new ArrayList<>(); + private int totalRowCount; + + public HyperRowSetBuilderImpl(BufferAllocator allocator) { + this.allocator = allocator; + } + + @Override + public void addBatch(SingleRowSet rowSet) { + if (rowSet.rowCount() == 0) { + return; + } + if (rowSet.indirectionType() != SelectionVectorMode.NONE) { + throw new IllegalArgumentException("Batches must not have a selection vector."); + } + batches.add(rowSet.container()); + totalRowCount += rowSet.rowCount(); + } + + @Override + public void addBatch(VectorContainer container) { + if (container.getRecordCount() == 0) { + return; + } + if (container.getSchema().getSelectionVectorMode() != SelectionVectorMode.NONE) { + throw new IllegalArgumentException("Batches must not have a selection vector."); + } + batches.add(container); + totalRowCount += container.getRecordCount(); + } + + @SuppressWarnings("resource") + @Override + public HyperRowSet build() throws SchemaChangeException { + SelectionVector4 sv4 = new SelectionVector4(allocator, totalRowCount); + ExpandableHyperContainer hyperContainer = new ExpandableHyperContainer(); + for (VectorContainer container : batches) { + hyperContainer.addBatch(container); + } + + // TODO: This has a bug. If the hyperset has two batches with unions, + // and the first union contains only VARCHAR, while the second contains + // only INT, the combined schema should be (VARCHAR, INT). Same is true + // of lists. But, this code looks at only the first container. + // + // This is only a theoretical bug as Drill does not support unions + // completely, but must be fixed if we want complete union support. + // + // Actually, the problem is more fundamental. The extendable hyper + // container, which creates the metadata schema, does not handle the + // case either. + + TupleMetadata schema = new HyperSchemaInference().infer(hyperContainer); + return new HyperRowSetImpl(schema, hyperContainer, sv4); } } @@ -56,13 +120,37 @@ public class HyperRowSetImpl extends AbstractRowSet implements HyperRowSet { private final SelectionVector4 sv4; - public HyperRowSetImpl(VectorContainer container, SelectionVector4 sv4) { - super(container, new SchemaInference().infer(container)); + public HyperRowSetImpl(TupleMetadata schema, VectorContainer container, SelectionVector4 sv4) { + super(container, schema); this.sv4 = sv4; } + public HyperRowSetImpl(VectorContainer container, SelectionVector4 sv4) throws SchemaChangeException { + this(new HyperSchemaInference().infer(container), container, sv4); + } + + public static HyperRowSetBuilder builder(BufferAllocator allocator) { + return new HyperRowSetBuilderImpl(allocator); + } + public static HyperRowSet fromContainer(VectorContainer container, SelectionVector4 sv4) { - return new HyperRowSetImpl(container, sv4); + try { + return new HyperRowSetImpl(container, sv4); + } catch (SchemaChangeException e) { + throw new UnsupportedOperationException(e); + } + } + + public static HyperRowSet fromRowSets(BufferAllocator allocator, SingleRowSet...rowSets) { + HyperRowSetBuilder builder = builder(allocator); + for (SingleRowSet rowSet : rowSets) { + builder.addBatch(rowSet); + } + try { + return builder.build(); + } catch (SchemaChangeException e) { + throw new IllegalArgumentException("Incompatible schemas", e); + } } @Override http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/IndirectRowSet.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/IndirectRowSet.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/IndirectRowSet.java index 42d97db..a2bc5e8 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/IndirectRowSet.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/IndirectRowSet.java @@ -20,10 +20,10 @@ package org.apache.drill.test.rowSet; import com.google.common.collect.Sets; import org.apache.drill.exec.exception.OutOfMemoryException; import org.apache.drill.exec.memory.BufferAllocator; -import org.apache.drill.exec.record.RecordBatchSizer; import org.apache.drill.exec.physical.rowSet.model.ReaderIndex; -import org.apache.drill.exec.physical.rowSet.model.SchemaInference; +import org.apache.drill.exec.physical.rowSet.model.single.SingleSchemaInference; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; +import org.apache.drill.exec.record.RecordBatchSizer; import org.apache.drill.exec.record.VectorContainer; import org.apache.drill.exec.record.selection.SelectionVector2; @@ -38,8 +38,8 @@ public class IndirectRowSet extends AbstractSingleRowSet { /** * Reader index that points to each row indirectly through the - * selection vector. The {@link #vectorIndex()} method points to the - * actual data row, while the {@link #position()} method gives + * selection vector. The {@link #physicalIndex()} method points to the + * actual data row, while the {@link #logicalIndex()} method gives * the position relative to the indirection vector. That is, * the position increases monotonically, but the index jumps * around as specified by the indirection vector. @@ -55,16 +55,16 @@ public class IndirectRowSet extends AbstractSingleRowSet { } @Override - public int vectorIndex() { return sv2.getIndex(rowIndex); } + public int offset() { return sv2.getIndex(position); } @Override - public int batchIndex() { return 0; } + public int hyperVectorIndex() { return 0; } } private final SelectionVector2 sv2; private IndirectRowSet(VectorContainer container, SelectionVector2 sv2) { - super(container, new SchemaInference().infer(container)); + super(container, new SingleSchemaInference().infer(container)); this.sv2 = sv2; } http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java index 53be75d..63a4959 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java @@ -17,6 +17,7 @@ */ package org.apache.drill.test.rowSet; +import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; @@ -143,4 +144,10 @@ public interface RowSet { interface HyperRowSet extends RowSet { SelectionVector4 getSv4(); } + + interface HyperRowSetBuilder { + void addBatch(SingleRowSet rowSet); + void addBatch(VectorContainer container); + HyperRowSet build() throws SchemaChangeException; + } } http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetComparison.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetComparison.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetComparison.java index 1cae64f..e098e33 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetComparison.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetComparison.java @@ -20,15 +20,14 @@ package org.apache.drill.test.rowSet; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.util.Comparator; + import org.apache.drill.exec.vector.accessor.ArrayReader; import org.apache.drill.exec.vector.accessor.ObjectReader; -import org.apache.drill.exec.vector.accessor.ScalarElementReader; import org.apache.drill.exec.vector.accessor.ScalarReader; import org.apache.drill.exec.vector.accessor.TupleReader; import org.bouncycastle.util.Arrays; -import java.util.Comparator; - /** * For testing, compare the contents of two row sets (record batches) * to verify that they are identical. Supports masks to exclude certain @@ -149,6 +148,9 @@ public class RowSetComparison { */ public void verify(RowSet actual) { + assertTrue("Schemas don't match.\n" + + "Expected: " + expected.schema().toString() + + "\nActual: " + actual.schema(), expected.schema().isEquivalent(actual.schema())); int testLength = expected.rowCount() - offset; if (span > -1) { testLength = span; @@ -165,7 +167,7 @@ public class RowSetComparison { for (int i = 0; i < testLength; i++) { er.next(); ar.next(); - String label = Integer.toString(er.index() + 1); + String label = Integer.toString(er.logicalIndex() + 1); verifyRow(label, er, ar); } } @@ -276,63 +278,12 @@ public class RowSetComparison { private void verifyArray(String label, ArrayReader ea, ArrayReader aa) { - assertEquals(label, ea.entryType(), aa.entryType()); - assertEquals(label, ea.size(), aa.size()); - switch (ea.entryType()) { - case ARRAY: - throw new UnsupportedOperationException(); - case SCALAR: - verifyScalarArray(label, ea.elements(), aa.elements()); - break; - case TUPLE: - verifyTupleArray(label, ea, aa); - break; - default: - throw new IllegalStateException( "Unexpected type: " + ea.entryType()); - } - } - - private void verifyTupleArray(String label, ArrayReader ea, ArrayReader aa) { - for (int i = 0; i < ea.size(); i++) { - verifyTuple(label + "[" + i + "]", ea.tuple(i), aa.tuple(i)); - } - } - - private void verifyScalarArray(String colLabel, ScalarElementReader ea, - ScalarElementReader aa) { - assertEquals(colLabel, ea.valueType(), aa.valueType()); - assertEquals(colLabel, ea.size(), aa.size()); - for (int i = 0; i < ea.size(); i++) { - String label = colLabel + "[" + i + "]"; - switch (ea.valueType()) { - case BYTES: { - byte expected[] = ea.getBytes(i); - byte actual[] = aa.getBytes(i); - assertEquals(label + " - byte lengths differ", expected.length, actual.length); - assertTrue(label, Arrays.areEqual(expected, actual)); - break; - } - case DOUBLE: - assertEquals(label, ea.getDouble(i), aa.getDouble(i), delta); - break; - case INTEGER: - assertEquals(label, ea.getInt(i), aa.getInt(i)); - break; - case LONG: - assertEquals(label, ea.getLong(i), aa.getLong(i)); - break; - case STRING: - assertEquals(label, ea.getString(i), aa.getString(i)); - break; - case DECIMAL: - assertEquals(label, ea.getDecimal(i), aa.getDecimal(i)); - break; - case PERIOD: - assertEquals(label, ea.getPeriod(i), aa.getPeriod(i)); - break; - default: - throw new IllegalStateException( "Unexpected type: " + ea.valueType()); - } + assertEquals(label + " - array element type", ea.entryType(), aa.entryType()); + assertEquals(label + " - array length", ea.size(), aa.size()); + int i = 0; + while (ea.next()) { + assertTrue(aa.next()); + verifyColumn(label + "[" + i++ + "]", ea.entry(), aa.entry()); } } http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java index fe50197..82d4bbd 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java @@ -41,7 +41,7 @@ public class RowSetPrinter { public void print(PrintStream out) { SelectionVectorMode selectionMode = rowSet.indirectionType(); RowSetReader reader = rowSet.reader(); - int colCount = reader.schema().size(); + int colCount = reader.tupleSchema().size(); printSchema(out, selectionMode, reader); while (reader.next()) { printHeader(out, reader, selectionMode); @@ -68,12 +68,12 @@ public class RowSetPrinter { break; } out.print(": "); - TupleMetadata schema = reader.schema(); + TupleMetadata schema = reader.tupleSchema(); printTupleSchema(out, schema); out.println(); } - private void printTupleSchema(PrintStream out, TupleMetadata schema) { + public static void printTupleSchema(PrintStream out, TupleMetadata schema) { for (int i = 0; i < schema.size(); i++) { if (i > 0) { out.print(", "); @@ -81,26 +81,26 @@ public class RowSetPrinter { ColumnMetadata colSchema = schema.metadata(i); out.print(colSchema.name()); if (colSchema.isMap()) { - out.print("("); + out.print("{"); printTupleSchema(out, colSchema.mapSchema()); - out.print(")"); + out.print("}"); } } } private void printHeader(PrintStream out, RowSetReader reader, SelectionVectorMode selectionMode) { - out.print(reader.index()); + out.print(reader.logicalIndex()); switch (selectionMode) { case FOUR_BYTE: out.print(" ("); - out.print(reader.batchIndex()); + out.print(reader.hyperVectorIndex()); out.print(", "); - out.print(reader.rowIndex()); + out.print(reader.offset()); out.print(")"); break; case TWO_BYTE: out.print(" ("); - out.print(reader.rowIndex()); + out.print(reader.offset()); out.print(")"); break; default: http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReader.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReader.java index 3e27529..4a966cd 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReader.java @@ -32,23 +32,22 @@ public interface RowSetReader extends TupleReader { int rowCount(); boolean next(); - int index(); - void set(int index); + int logicalIndex(); + void setPosn(int index); /** * Batch index: 0 for a single batch, batch for the current * row is a hyper-batch. * @return index of the batch for the current row */ - int batchIndex(); + int hyperVectorIndex(); /** * The index of the underlying row which may be indexed by an - * Sv2 or Sv4. + * SV2 or SV4. * * @return */ - int rowIndex(); - boolean valid(); + int offset(); } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReaderImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReaderImpl.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReaderImpl.java index 7217187..6344419 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReaderImpl.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReaderImpl.java @@ -20,9 +20,11 @@ package org.apache.drill.test.rowSet; import java.util.List; import org.apache.drill.exec.physical.rowSet.model.ReaderIndex; +import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.accessor.reader.AbstractObjectReader; import org.apache.drill.exec.vector.accessor.reader.AbstractTupleReader; +import org.apache.drill.exec.vector.accessor.reader.NullStateReaders; /** * Reader implementation for a row set. @@ -30,11 +32,14 @@ import org.apache.drill.exec.vector.accessor.reader.AbstractTupleReader; public class RowSetReaderImpl extends AbstractTupleReader implements RowSetReader { + private final TupleMetadata schema; protected final ReaderIndex readerIndex; public RowSetReaderImpl(TupleMetadata schema, ReaderIndex index, AbstractObjectReader[] readers) { - super(schema, readers); + super(readers); + this.schema = schema; this.readerIndex = index; + bindNullState(NullStateReaders.REQUIRED_STATE_READER); bindIndex(index); } @@ -54,23 +59,26 @@ public class RowSetReaderImpl extends AbstractTupleReader implements RowSetReade } @Override - public boolean valid() { return readerIndex.valid(); } - - @Override - public int index() { return readerIndex.position(); } + public int logicalIndex() { return readerIndex.logicalIndex(); } @Override public int rowCount() { return readerIndex.size(); } @Override - public int rowIndex() { return readerIndex.vectorIndex(); } + public int offset() { return readerIndex.offset(); } @Override - public int batchIndex() { return readerIndex.batchIndex(); } + public int hyperVectorIndex() { return readerIndex.hyperVectorIndex(); } @Override - public void set(int index) { + public void setPosn(int index) { this.readerIndex.set(index); reposition(); } + + @Override + public ColumnMetadata schema() { return null; } + + @Override + public TupleMetadata tupleSchema() { return schema; } } http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/UnionBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/UnionBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/UnionBuilder.java index e47e552..1aa636d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/UnionBuilder.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/UnionBuilder.java @@ -17,9 +17,9 @@ */ package org.apache.drill.test.rowSet.schema; -import org.apache.drill.common.types.Types; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.common.types.Types; import org.apache.drill.exec.record.metadata.AbstractColumnMetadata; import org.apache.drill.exec.record.metadata.VariantColumnMetadata; import org.apache.drill.exec.record.metadata.VariantSchema; @@ -33,7 +33,6 @@ public class UnionBuilder implements SchemaContainer { private final SchemaContainer parent; private final String name; private final MinorType type; - private final DataMode mode; private final VariantSchema union; public UnionBuilder(SchemaContainer parent, String name, @@ -41,7 +40,6 @@ public class UnionBuilder implements SchemaContainer { this.parent = parent; this.name = name; this.type = type; - this.mode = mode; union = new VariantSchema(); } http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java index 9ad4133..92b9266 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java @@ -36,7 +36,6 @@ import org.apache.drill.exec.vector.VectorOverflowException; import org.apache.drill.exec.vector.accessor.ArrayReader; import org.apache.drill.exec.vector.accessor.ArrayWriter; import org.apache.drill.exec.vector.accessor.ObjectType; -import org.apache.drill.exec.vector.accessor.ScalarElementReader; import org.apache.drill.exec.vector.accessor.ScalarReader; import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.apache.drill.exec.vector.accessor.TupleReader; @@ -45,6 +44,7 @@ import org.apache.drill.exec.vector.accessor.ValueType; import org.apache.drill.exec.vector.complex.MapVector; import org.apache.drill.exec.vector.complex.RepeatedMapVector; import org.apache.drill.test.SubOperatorTest; +import org.apache.drill.test.rowSet.DirectRowSet; import org.apache.drill.test.rowSet.RowSet.ExtendableRowSet; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.apache.drill.test.rowSet.RowSetComparison; @@ -60,12 +60,17 @@ import org.junit.Test; * Tests basic protocol of the writers: <pre><code> * row : tuple * tuple : column * - * column : scalar obj | array obj | tuple obj + * column : scalar obj | array obj | tuple obj | variant obj * scalar obj : scalar - * array obj : array writer - * array writer : element + * array obj : array + * array : index --> column * element : column - * tuple obj : tuple</code></pre> + * tuple obj : tuple + * tuple : name --> column (also index --> column) + * variant obj : variant + * variant : type --> column</code></pre> + * <p> + * A list is an array of variants. Variants are tested elsewhere. */ public class RowSetTest extends SubOperatorTest { @@ -137,16 +142,21 @@ public class RowSetTest extends SubOperatorTest { assertSame(reader.column("a").scalar(), reader.scalar("a")); assertSame(reader.column(0).scalar(), reader.scalar(0)); assertEquals(ValueType.INTEGER, reader.scalar(0).valueType()); + assertTrue(schema.metadata("a").isEquivalent(reader.column("a").schema())); // Test various accessors: full and simple assertTrue(reader.next()); + assertFalse(reader.column("a").scalar().isNull()); assertEquals(10, reader.column("a").scalar().getInt()); assertTrue(reader.next()); + assertFalse(reader.scalar("a").isNull()); assertEquals(20, reader.scalar("a").getInt()); assertTrue(reader.next()); + assertFalse(reader.column(0).scalar().isNull()); assertEquals(30, reader.column(0).scalar().getInt()); assertTrue(reader.next()); + assertFalse(reader.column(0).scalar().isNull()); assertEquals(40, reader.scalar(0).getInt()); assertFalse(reader.next()); @@ -238,27 +248,50 @@ public class RowSetTest extends SubOperatorTest { assertSame(reader.column(0).array(), reader.array(0)); assertEquals(ObjectType.SCALAR, reader.column("a").array().entryType()); - assertEquals(ValueType.INTEGER, reader.array(0).elements().valueType()); + assertEquals(ValueType.INTEGER, reader.array(0).scalar().valueType()); // Read and verify the rows - ScalarElementReader intReader = reader.array(0).elements(); + ArrayReader arrayReader = reader.array(0); + ScalarReader intReader = arrayReader.scalar(); assertTrue(reader.next()); - assertEquals(2, intReader.size()); - assertEquals(10, intReader.getInt(0)); - assertEquals(11, intReader.getInt(1)); + assertFalse(arrayReader.isNull()); + assertEquals(2, arrayReader.size()); + + assertTrue(arrayReader.next()); + assertEquals(10, intReader.getInt()); + assertTrue(arrayReader.next()); + assertEquals(11, intReader.getInt()); + assertFalse(arrayReader.next()); + assertTrue(reader.next()); - assertEquals(3, intReader.size()); - assertEquals(20, intReader.getInt(0)); - assertEquals(21, intReader.getInt(1)); - assertEquals(22, intReader.getInt(2)); + assertFalse(arrayReader.isNull()); + assertEquals(3, arrayReader.size()); + + assertTrue(arrayReader.next()); + assertEquals(20, intReader.getInt()); + assertTrue(arrayReader.next()); + assertEquals(21, intReader.getInt()); + assertTrue(arrayReader.next()); + assertEquals(22, intReader.getInt()); + assertFalse(arrayReader.next()); + assertTrue(reader.next()); - assertEquals(1, intReader.size()); - assertEquals(30, intReader.getInt(0)); + assertFalse(arrayReader.isNull()); + assertEquals(1, arrayReader.size()); + assertTrue(arrayReader.next()); + assertEquals(30, intReader.getInt()); + assertFalse(arrayReader.next()); + assertTrue(reader.next()); - assertEquals(2, intReader.size()); - assertEquals(40, intReader.getInt(0)); - assertEquals(41, intReader.getInt(1)); + assertFalse(arrayReader.isNull()); + assertEquals(2, arrayReader.size()); + assertTrue(arrayReader.next()); + assertEquals(40, intReader.getInt()); + assertTrue(arrayReader.next()); + assertEquals(41, intReader.getInt()); + assertFalse(arrayReader.next()); + assertFalse(reader.next()); // Test the above again via the writer and reader @@ -352,22 +385,47 @@ public class RowSetTest extends SubOperatorTest { ScalarReader aReader = reader.column(0).scalar(); TupleReader mReader = reader.column(1).tuple(); - assertEquals(ObjectType.SCALAR, mReader.column("b").array().entryType()); - ScalarElementReader bReader = mReader.column(0).elements(); + ArrayReader bArray = mReader.column("b").array(); + assertEquals(ObjectType.SCALAR, bArray.entryType()); + ScalarReader bReader = bArray.scalar(); assertEquals(ValueType.INTEGER, bReader.valueType()); + // Row 1: (10, {[11, 12]}) + assertTrue(reader.next()); assertEquals(10, aReader.getInt()); - assertEquals(11, bReader.getInt(0)); - assertEquals(12, bReader.getInt(1)); + assertFalse(mReader.isNull()); + + assertTrue(bArray.next()); + assertFalse(bReader.isNull()); + assertEquals(11, bReader.getInt()); + assertTrue(bArray.next()); + assertFalse(bReader.isNull()); + assertEquals(12, bReader.getInt()); + assertFalse(bArray.next()); + + // Row 2: (20, {[21, 22]}) + assertTrue(reader.next()); assertEquals(20, aReader.getInt()); - assertEquals(21, bReader.getInt(0)); - assertEquals(22, bReader.getInt(1)); + assertFalse(mReader.isNull()); + + assertTrue(bArray.next()); + assertEquals(21, bReader.getInt()); + assertTrue(bArray.next()); + assertEquals(22, bReader.getInt()); + + // Row 3: (30, {[31, 32]}) + assertTrue(reader.next()); assertEquals(30, aReader.getInt()); - assertEquals(31, bReader.getInt(0)); - assertEquals(32, bReader.getInt(1)); + assertFalse(mReader.isNull()); + + assertTrue(bArray.next()); + assertEquals(31, bReader.getInt()); + assertTrue(bArray.next()); + assertEquals(32, bReader.getInt()); + assertFalse(reader.next()); // Verify that the map accessor's value count was set. @@ -472,18 +530,22 @@ public class RowSetTest extends SubOperatorTest { assertEquals(ValueType.INTEGER, bReader.valueType()); assertEquals(ValueType.INTEGER, cReader.valueType()); - // Row 1: use index accessors + // Row 1: Use iterator-like accessors assertTrue(reader.next()); assertEquals(10, aReader.getInt()); - TupleReader ixReader = maReader.tuple(0); - assertEquals(101, ixReader.scalar(0).getInt()); - assertEquals(102, ixReader.scalar(1).getInt()); - ixReader = maReader.tuple(1); - assertEquals(111, ixReader.scalar(0).getInt()); - assertEquals(112, ixReader.scalar(1).getInt()); - - // Row 2: use common accessor with explicit positioning, + assertFalse(maReader.isNull()); // Array itself is not null + + assertTrue(maReader.next()); + assertFalse(mapReader.isNull()); // Tuple 0 is not null + assertEquals(101, mapReader.scalar(0).getInt()); + assertEquals(102, mapReader.scalar(1).getInt()); + + assertTrue(maReader.next()); + assertEquals(111, mapReader.scalar(0).getInt()); + assertEquals(112, mapReader.scalar(1).getInt()); + + // Row 2: use explicit positioning, // but access scalars through the map reader. assertTrue(reader.next()); @@ -495,14 +557,16 @@ public class RowSetTest extends SubOperatorTest { assertEquals(211, mapReader.scalar(0).getInt()); assertEquals(212, mapReader.scalar(1).getInt()); - // Row 3: use common accessor for scalars + // Row 3: use scalar accessor assertTrue(reader.next()); assertEquals(30, aReader.getInt()); - maReader.setPosn(0); + + assertTrue(maReader.next()); assertEquals(301, bReader.getInt()); assertEquals(302, cReader.getInt()); - maReader.setPosn(1); + + assertTrue(maReader.next()); assertEquals(311, bReader.getInt()); assertEquals(312, cReader.getInt()); @@ -555,18 +619,28 @@ public class RowSetTest extends SubOperatorTest { SingleRowSet result = writer.done(); RowSetReader reader = result.reader(); + ArrayReader arrayReader = reader.array(1); + ScalarReader elementReader = arrayReader.scalar(); + assertTrue(reader.next()); assertEquals(10, reader.scalar(0).getInt()); - ScalarElementReader arrayReader = reader.array(1).elements(); assertEquals(2, arrayReader.size()); - assertEquals(100, arrayReader.getInt(0)); - assertEquals(110, arrayReader.getInt(1)); + + assertTrue(arrayReader.next()); + assertEquals(100, elementReader.getInt()); + assertTrue(arrayReader.next()); + assertEquals(110, elementReader.getInt()); + assertTrue(reader.next()); assertEquals(20, reader.scalar(0).getInt()); assertEquals(3, arrayReader.size()); - assertEquals(200, arrayReader.getInt(0)); - assertEquals(120, arrayReader.getInt(1)); - assertEquals(220, arrayReader.getInt(2)); + assertTrue(arrayReader.next()); + assertEquals(200, elementReader.getInt()); + assertTrue(arrayReader.next()); + assertEquals(120, elementReader.getInt()); + assertTrue(arrayReader.next()); + assertEquals(220, elementReader.getInt()); + assertTrue(reader.next()); assertEquals(30, reader.scalar(0).getInt()); assertEquals(0, arrayReader.size()); @@ -581,7 +655,6 @@ public class RowSetTest extends SubOperatorTest { new RowSetComparison(rs1) .verifyAndClearAll(rs2); } - /** * Test filling a row set up to the maximum number of rows. * Values are small enough to prevent filling to the @@ -666,4 +739,107 @@ public class RowSetTest extends SubOperatorTest { assertEquals(count, rs.rowCount()); rs.clear(); } + + /** + * The code below is not a test. Rather, it is a simple example of + * how to write a batch of data using writers, then read it using + * readers. + */ + + @Test + public void example() { + + // Step 1: Define a schema. In a real app, this + // will be provided by a reader, by an incoming batch, + // etc. + + BatchSchema schema = new SchemaBuilder() + .add("a", MinorType.VARCHAR) + .addArray("b", MinorType.INT) + .addMap("c") + .add("c1", MinorType.INT) + .add("c2", MinorType.VARCHAR) + .resumeSchema() + .build(); + + // Step 2: Create a batch. Done here because this is + // a batch-oriented test. Done automatically in the + // result set loader. + + DirectRowSet drs = DirectRowSet.fromSchema(fixture.allocator(), schema); + + // Step 3: Create the writer. + + RowSetWriter writer = drs.writer(); + + // Step 4: Populate data. Here we do it the way an app would: + // using the individual accessors. See tests above for the many + // ways this can be done depending on the need of the app. + // + // Write two rows: + // ("fred", [10, 11], {12, "wilma"}) + // ("barney", [20, 21], {22, "betty"}) + // + // This example uses Java strings for Varchar. Real code might + // use byte arrays. + + writer.scalar("a").setString("fred"); + ArrayWriter bWriter = writer.array("b"); + bWriter.scalar().setInt(10); + bWriter.scalar().setInt(11); + TupleWriter cWriter = writer.tuple("c"); + cWriter.scalar("c1").setInt(12); + cWriter.scalar("c2").setString("wilma"); + writer.save(); + + writer.scalar("a").setString("barney"); + bWriter.scalar().setInt(20); + bWriter.scalar().setInt(21); + cWriter.scalar("c1").setInt(22); + cWriter.scalar("c2").setString("betty"); + writer.save(); + + // Step 5: "Harvest" the batch. Done differently in the + // result set loader. + + SingleRowSet rowSet = writer.done(); + + // Step 5: Create a reader. + + RowSetReader reader = rowSet.reader(); + + // Step 6: Retrieve the data. Here we just print the + // values. + + while (reader.next()) { + print(reader.scalar("a").getString()); + ArrayReader bReader = reader.array("b"); + while (bReader.next()) { + print(bReader.scalar().getInt()); + } + TupleReader cReader = reader.tuple("c"); + print(cReader.scalar("c1").getInt()); + print(cReader.scalar("c2").getString()); + endRow(); + } + + // Step 7: Free memory. + + rowSet.clear(); + } + + public void print(Object obj) { + if (obj instanceof String) { + System.out.print("\""); + System.out.print(obj); + System.out.print("\""); + } else { + System.out.print(obj); + } + System.out.print(" "); + } + + public void endRow() { + System.out.println(); + } } http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java index 490a9fd..f4c00ca 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java @@ -24,17 +24,17 @@ import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.metadata.TupleMetadata; -import org.apache.drill.exec.vector.accessor.ScalarElementReader; +import org.apache.drill.exec.vector.accessor.ArrayReader; import org.apache.drill.exec.vector.accessor.ScalarReader; import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.apache.drill.exec.vector.accessor.ValueType; import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.RowSet.ExtendableRowSet; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; -import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.apache.drill.test.rowSet.RowSetReader; import org.apache.drill.test.rowSet.RowSetUtilities; import org.apache.drill.test.rowSet.RowSetWriter; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.junit.Test; /** @@ -219,16 +219,18 @@ public class TestFillEmpties extends SubOperatorTest { } SingleRowSet result = writer.done(); RowSetReader reader = result.reader(); - ScalarElementReader colReader = reader.array(0).elements(); + ArrayReader aReader = reader.array(0); + ScalarReader colReader = aReader.scalar(); for (int i = 0; i < ROW_COUNT; i++) { assertTrue(reader.next()); if (i % 5 != 0) { // Empty arrays are defined to be the same as a zero-length array. - assertEquals(0, colReader.size()); + assertEquals(0, aReader.size()); } else { for (int j = 0; j < 2; j++) { - Object actual = colReader.getObject(j); + assertTrue(aReader.next()); + Object actual = colReader.getObject(); Object expected = RowSetUtilities.testDataFromInt(valueType, majorType, i + j); RowSetUtilities.assertEqualValues( majorType.toString().replace('\n', ' ') + "[" + i + "][" + j + "]", http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestHyperVectorReaders.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestHyperVectorReaders.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestHyperVectorReaders.java new file mode 100644 index 0000000..00e72d4 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestHyperVectorReaders.java @@ -0,0 +1,365 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.test.rowSet.test; + +import static org.apache.drill.test.rowSet.RowSetUtilities.mapArray; +import static org.apache.drill.test.rowSet.RowSetUtilities.mapValue; +import static org.apache.drill.test.rowSet.RowSetUtilities.strArray; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.exec.record.selection.SelectionVector4; +import org.apache.drill.test.SubOperatorTest; +import org.apache.drill.test.rowSet.HyperRowSetImpl; +import org.apache.drill.test.rowSet.RowSet.ExtendableRowSet; +import org.apache.drill.test.rowSet.RowSet.HyperRowSet; +import org.apache.drill.test.rowSet.RowSet.SingleRowSet; +import org.apache.drill.test.rowSet.RowSetBuilder; +import org.apache.drill.test.rowSet.RowSetReader; +import org.apache.drill.test.rowSet.RowSetUtilities; +import org.apache.drill.test.rowSet.RowSetWriter; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; +import org.junit.Test; + +/** + * Test the reader mechanism that reads rows indexed via an SV4. + * SV4's introduce an additional level of indexing: each row may + * come from a different batch. The readers use the SV4 to find + * the root batch and vector, then must navigate downward from that + * vector for maps, repeated maps, lists, unions, repeated lists, + * nullable vectors and variable-length vectors. + * <p> + * This test does not cover repeated vectors; those tests should be added. + */ + +public class TestHyperVectorReaders extends SubOperatorTest { + + /** + * Test the simplest case: a top-level required vector. Has no contained vectors. + * This test focuses on the SV4 indirection mechanism itself. + */ + + @Test + public void testRequired() { + TupleMetadata schema = new SchemaBuilder() + .add("a", MinorType.INT) + .buildSchema(); + + SingleRowSet rowSet1; + { + ExtendableRowSet rowSet = fixture.rowSet(schema); + RowSetWriter writer = rowSet.writer(); + for (int i = 0; i < 10; i++) { + writer.scalar(0).setInt(i * 10); + writer.save(); + } + rowSet1 = writer.done(); + } + + SingleRowSet rowSet2; + { + ExtendableRowSet rowSet = fixture.rowSet(schema); + RowSetWriter writer = rowSet.writer(); + for (int i = 10; i < 20; i++) { + writer.scalar(0).setInt(i * 10); + writer.save(); + } + rowSet2 = writer.done(); + } + + // Build the hyper batch + // [0, 10, 20, ... 190] + + HyperRowSet hyperSet = HyperRowSetImpl.fromRowSets(fixture.allocator(), rowSet1, rowSet2); + assertEquals(20, hyperSet.rowCount()); + + // Populate the indirection vector: + // (1, 9), (0, 9), (1, 8), (0, 8), ... (0, 0) + + SelectionVector4 sv4 = hyperSet.getSv4(); + for (int i = 0; i < 20; i++) { + int batch = i % 2; + int offset = 9 - i / 2; + sv4.set(i, batch, offset); + } + + // Sanity check. + + for (int i = 0; i < 20; i++) { + int batch = i % 2; + int offset = 9 - i / 2; + int encoded = sv4.get(i); + assertEquals(batch, SelectionVector4.getBatchIndex(encoded)); + assertEquals(offset, SelectionVector4.getRecordIndex(encoded)); + } + + // Verify reader + // Expected: [190, 90, 180, 80, ... 0] + + RowSetReader reader = hyperSet.reader(); + for (int i = 0; i < 20; i++) { + assertTrue(reader.next()); + int batch = i % 2; + int offset = 9 - i / 2; + int expected = batch * 100 + offset * 10; + assertEquals(expected, reader.scalar(0).getInt()); + } + assertFalse(reader.next()); + + // Validate using an expected result set. + + RowSetBuilder rsBuilder = fixture.rowSetBuilder(schema); + for (int i = 0; i < 20; i++) { + int batch = i % 2; + int offset = 9 - i / 2; + int expected = batch * 100 + offset * 10; + rsBuilder.addRow(expected); + } + + RowSetUtilities.verify(rsBuilder.build(), hyperSet); + } + + @Test + public void testVarWidth() { + TupleMetadata schema = new SchemaBuilder() + .add("a", MinorType.VARCHAR) + .buildSchema(); + + SingleRowSet rowSet1 = fixture.rowSetBuilder(schema) + .addSingleCol("second") + .addSingleCol("fourth") + .build(); + + SingleRowSet rowSet2 = fixture.rowSetBuilder(schema) + .addSingleCol("first") + .addSingleCol("third") + .build(); + + // Build the hyper batch + + HyperRowSet hyperSet = HyperRowSetImpl.fromRowSets(fixture.allocator(), rowSet1, rowSet2); + assertEquals(4, hyperSet.rowCount()); + SelectionVector4 sv4 = hyperSet.getSv4(); + sv4.set(0, 1, 0); + sv4.set(1, 0, 0); + sv4.set(2, 1, 1); + sv4.set(3, 0, 1); + + SingleRowSet expected = fixture.rowSetBuilder(schema) + .addRow("first") + .addRow("second") + .addRow("third") + .addRow("fourth") + .build(); + + RowSetUtilities.verify(expected, hyperSet); + } + + /** + * Test a nullable varchar. Requires multiple indirections: + * <ul> + * <li>From the SV4 to the nullable vector.</li> + * <li>From the nullable vector to the bits vector.</li> + * <li>From the nullable vector to the data vector.</li> + * <li>From the data vector to the offset vector.</li> + * <li>From the data vector to the values vector.</li> + * </ul> + * All are coordinated by the vector index and vector accessors. + * This test verifies that each of the indirections does, in fact, + * work as expected. + */ + + @Test + public void testOptional() { + TupleMetadata schema = new SchemaBuilder() + .addNullable("a", MinorType.VARCHAR) + .buildSchema(); + + SingleRowSet rowSet1 = fixture.rowSetBuilder(schema) + .addSingleCol("sixth") + .addSingleCol(null) + .addSingleCol("fourth") + .build(); + + SingleRowSet rowSet2 = fixture.rowSetBuilder(schema) + .addSingleCol(null) + .addSingleCol("first") + .addSingleCol("third") + .build(); + + // Build the hyper batch + + HyperRowSet hyperSet = HyperRowSetImpl.fromRowSets(fixture.allocator(), rowSet1, rowSet2); + assertEquals(6, hyperSet.rowCount()); + SelectionVector4 sv4 = hyperSet.getSv4(); + sv4.set(0, 1, 1); + sv4.set(1, 0, 1); + sv4.set(2, 1, 2); + sv4.set(3, 0, 2); + sv4.set(4, 1, 0); + sv4.set(5, 0, 0); + + SingleRowSet expected = fixture.rowSetBuilder(schema) + .addSingleCol("first") + .addSingleCol(null) + .addSingleCol("third") + .addSingleCol("fourth") + .addSingleCol(null) + .addSingleCol("sixth") + .build(); + + RowSetUtilities.verify(expected, hyperSet); + } + + /** + * Test an array to test the indirection from the repeated vector + * to the array offsets vector and the array values vector. (Uses + * varchar to add another level of indirection to the data offset + * and data values vectors.) + */ + + @Test + public void testRepeated() { + TupleMetadata schema = new SchemaBuilder() + .addArray("a", MinorType.VARCHAR) + .buildSchema(); + + SingleRowSet rowSet1 = fixture.rowSetBuilder(schema) + .addSingleCol(strArray("sixth", "6.1", "6.2")) + .addSingleCol(strArray("second", "2.1", "2.2", "2.3")) + .addSingleCol(strArray("fourth", "4.1")) + .build(); + + SingleRowSet rowSet2 = fixture.rowSetBuilder(schema) + .addSingleCol(strArray("fifth", "51", "5.2")) + .addSingleCol(strArray("first", "1.1", "1.2", "1.3")) + .addSingleCol(strArray("third", "3.1")) + .build(); + + // Build the hyper batch + + HyperRowSet hyperSet = HyperRowSetImpl.fromRowSets(fixture.allocator(), rowSet1, rowSet2); + assertEquals(6, hyperSet.rowCount()); + SelectionVector4 sv4 = hyperSet.getSv4(); + sv4.set(0, 1, 1); + sv4.set(1, 0, 1); + sv4.set(2, 1, 2); + sv4.set(3, 0, 2); + sv4.set(4, 1, 0); + sv4.set(5, 0, 0); + + SingleRowSet expected = fixture.rowSetBuilder(schema) + .addSingleCol(strArray("first", "1.1", "1.2", "1.3")) + .addSingleCol(strArray("second", "2.1", "2.2", "2.3")) + .addSingleCol(strArray("third", "3.1")) + .addSingleCol(strArray("fourth", "4.1")) + .addSingleCol(strArray("fifth", "51", "5.2")) + .addSingleCol(strArray("sixth", "6.1", "6.2")) + .build(); + + RowSetUtilities.verify(expected, hyperSet); + } + + /** + * Maps are an interesting case. The hyper-vector wrapper holds a mirror-image of the + * map members. So, we can reach the map members either via the vector wrappers or + * the original map vector. + */ + + @Test + public void testMap() { + TupleMetadata schema = new SchemaBuilder() + .addMap("m") + .add("a", MinorType.INT) + .add("b", MinorType.VARCHAR) + .resumeSchema() + .buildSchema(); + + SingleRowSet rowSet1 = fixture.rowSetBuilder(schema) + .addSingleCol(mapValue(2, "second")) + .addSingleCol(mapValue(4, "fourth")) + .build(); + + SingleRowSet rowSet2 = fixture.rowSetBuilder(schema) + .addSingleCol(mapValue(2, "first")) + .addSingleCol(mapValue(4, "third")) + .build(); + + // Build the hyper batch + + HyperRowSet hyperSet = HyperRowSetImpl.fromRowSets(fixture.allocator(), rowSet1, rowSet2); + assertEquals(4, hyperSet.rowCount()); + SelectionVector4 sv4 = hyperSet.getSv4(); + sv4.set(0, 1, 0); + sv4.set(1, 0, 0); + sv4.set(2, 1, 1); + sv4.set(3, 0, 1); + + SingleRowSet expected = fixture.rowSetBuilder(schema) + .addSingleCol(mapValue(2, "first")) + .addSingleCol(mapValue(2, "second")) + .addSingleCol(mapValue(4, "third")) + .addSingleCol(mapValue(4, "fourth")) + .build(); + + RowSetUtilities.verify(expected, hyperSet); + } + + @Test + public void testRepeatedMap() { + TupleMetadata schema = new SchemaBuilder() + .add("a", MinorType.INT) + .addMapArray("ma") + .add("b", MinorType.INT) + .add("c", MinorType.VARCHAR) + .resumeSchema() + .buildSchema(); + + SingleRowSet rowSet1 = fixture.rowSetBuilder(schema) + .addRow(2, mapArray(mapValue(21, "second.1"), mapValue(22, "second.2"))) + .addRow(4, mapArray(mapValue(41, "fourth.1"))) + .build(); + + SingleRowSet rowSet2 = fixture.rowSetBuilder(schema) + .addRow(1, mapArray(mapValue(11, "first.1"), mapValue(12, "first.2"))) + .addRow(3, mapArray(mapValue(31, "third.1"), mapValue(32, "third.2"), mapValue(33, "third.3"))) + .build(); + + // Build the hyper batch + + HyperRowSet hyperSet = HyperRowSetImpl.fromRowSets(fixture.allocator(), rowSet1, rowSet2); + assertEquals(4, hyperSet.rowCount()); + SelectionVector4 sv4 = hyperSet.getSv4(); + sv4.set(0, 1, 0); + sv4.set(1, 0, 0); + sv4.set(2, 1, 1); + sv4.set(3, 0, 1); + + SingleRowSet expected = fixture.rowSetBuilder(schema) + .addRow(1, mapArray(mapValue(11, "first.1"), mapValue(12, "first.2"))) + .addRow(2, mapArray(mapValue(21, "second.1"), mapValue(22, "second.2"))) + .addRow(3, mapArray(mapValue(31, "third.1"), mapValue(32, "third.2"), mapValue(33, "third.3"))) + .addRow(4, mapArray(mapValue(41, "fourth.1"))) + .build(); + + RowSetUtilities.verify(expected, hyperSet); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java index 1daeef4..ecd1ebc 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java @@ -26,7 +26,8 @@ import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.BatchSchema; -import org.apache.drill.exec.vector.accessor.ScalarElementReader; +import org.apache.drill.exec.vector.DateUtilities; +import org.apache.drill.exec.vector.accessor.ArrayReader; import org.apache.drill.exec.vector.accessor.ScalarReader; import org.apache.drill.exec.vector.accessor.ValueType; import org.apache.drill.test.SubOperatorTest; @@ -36,6 +37,8 @@ import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.junit.Test; +import com.google.common.collect.Lists; + /** * Verify that simple scalar (non-repeated) column readers * and writers work as expected. The focus is on the generated @@ -51,6 +54,70 @@ import org.junit.Test; public class TestScalarAccessors extends SubOperatorTest { @Test + public void testUInt1RW() { + BatchSchema batchSchema = new SchemaBuilder() + .add("col", MinorType.UINT1) + .build(); + SingleRowSet rs = fixture.rowSetBuilder(batchSchema) + .addRow(0) + .addRow(0x7F) + .addRow(0xFF) + .build(); + assertEquals(3, rs.rowCount()); + + RowSetReader reader = rs.reader(); + ScalarReader colReader = reader.scalar(0); + assertEquals(ValueType.INTEGER, colReader.valueType()); + + assertTrue(reader.next()); + assertFalse(colReader.isNull()); + assertEquals(0, colReader.getInt()); + + assertTrue(reader.next()); + assertEquals(0x7F, colReader.getInt()); + assertEquals(0x7F, colReader.getObject()); + assertEquals(Integer.toString(0x7F), colReader.getAsString()); + + assertTrue(reader.next()); + assertEquals(0xFF, colReader.getInt()); + + assertFalse(reader.next()); + rs.clear(); + } + + @Test + public void testUInt2RW() { + BatchSchema batchSchema = new SchemaBuilder() + .add("col", MinorType.UINT2) + .build(); + SingleRowSet rs = fixture.rowSetBuilder(batchSchema) + .addRow(0) + .addRow(0x7FFF) + .addRow(0xFFFF) + .build(); + assertEquals(3, rs.rowCount()); + + RowSetReader reader = rs.reader(); + ScalarReader colReader = reader.scalar(0); + assertEquals(ValueType.INTEGER, colReader.valueType()); + + assertTrue(reader.next()); + assertFalse(colReader.isNull()); + assertEquals(0, colReader.getInt()); + + assertTrue(reader.next()); + assertEquals(0x7FFF, colReader.getInt()); + assertEquals(0x7FFF, colReader.getObject()); + assertEquals(Integer.toString(0x7FFF), colReader.getAsString()); + + assertTrue(reader.next()); + assertEquals(0xFFFF, colReader.getInt()); + + assertFalse(reader.next()); + rs.clear(); + } + + @Test public void testTinyIntRW() { BatchSchema batchSchema = new SchemaBuilder() .add("col", MinorType.TINYINT) @@ -129,23 +196,37 @@ public class TestScalarAccessors extends SubOperatorTest { assertEquals(2, rs.rowCount()); RowSetReader reader = rs.reader(); - ScalarElementReader colReader = reader.elements(0); + ArrayReader arrayReader = reader.array(0); + ScalarReader colReader = arrayReader.scalar(); assertEquals(ValueType.INTEGER, colReader.valueType()); assertTrue(reader.next()); - assertEquals(0, colReader.size()); + assertEquals(0, arrayReader.size()); assertTrue(reader.next()); - assertEquals(3, colReader.size()); - assertEquals(0, colReader.getInt(0)); - assertEquals(20, colReader.getInt(1)); - assertEquals(30, colReader.getInt(2)); - assertEquals(0, colReader.getObject(0)); - assertEquals(20, colReader.getObject(1)); - assertEquals(30, colReader.getObject(2)); - assertEquals("0", colReader.getAsString(0)); - assertEquals("20", colReader.getAsString(1)); - assertEquals("30", colReader.getAsString(2)); + assertEquals(3, arrayReader.size()); + assertTrue(arrayReader.next()); + assertFalse(colReader.isNull()); + assertEquals(0, colReader.getInt()); + assertEquals(0, colReader.getObject()); + assertEquals("0", colReader.getAsString()); + + assertTrue(arrayReader.next()); + assertFalse(colReader.isNull()); + assertEquals(20, colReader.getInt()); + assertEquals(20, colReader.getObject()); + assertEquals("20", colReader.getAsString()); + + assertTrue(arrayReader.next()); + assertFalse(colReader.isNull()); + assertEquals(30, colReader.getInt()); + assertEquals(30, colReader.getObject()); + assertEquals("30", colReader.getAsString()); + + assertFalse(arrayReader.next()); + + assertEquals("[0, 20, 30]", arrayReader.getAsString()); + assertEquals(Lists.newArrayList(0, 20, 30), arrayReader.getObject()); assertFalse(reader.next()); rs.clear(); @@ -323,23 +404,35 @@ public class TestScalarAccessors extends SubOperatorTest { assertEquals(2, rs.rowCount()); RowSetReader reader = rs.reader(); - ScalarElementReader colReader = reader.elements(0); + ArrayReader arrayReader = reader.array(0); + ScalarReader colReader = arrayReader.scalar(); assertEquals(ValueType.LONG, colReader.valueType()); assertTrue(reader.next()); - assertEquals(0, colReader.size()); + assertEquals(0, arrayReader.size()); assertTrue(reader.next()); - assertEquals(3, colReader.size()); - assertEquals(0, colReader.getLong(0)); - assertEquals(20, colReader.getLong(1)); - assertEquals(30, colReader.getLong(2)); - assertEquals(0L, colReader.getObject(0)); - assertEquals(20L, colReader.getObject(1)); - assertEquals(30L, colReader.getObject(2)); - assertEquals("0", colReader.getAsString(0)); - assertEquals("20", colReader.getAsString(1)); - assertEquals("30", colReader.getAsString(2)); + assertEquals(3, arrayReader.size()); + + assertTrue(arrayReader.next()); + assertEquals(0, colReader.getLong()); + assertEquals(0L, colReader.getObject()); + assertEquals("0", colReader.getAsString()); + + assertTrue(arrayReader.next()); + assertEquals(20, colReader.getLong()); + assertEquals(20L, colReader.getObject()); + assertEquals("20", colReader.getAsString()); + + assertTrue(arrayReader.next()); + assertEquals(30, colReader.getLong()); + assertEquals(30L, colReader.getObject()); + assertEquals("30", colReader.getAsString()); + + assertFalse(arrayReader.next()); + + assertEquals("[0, 20, 30]", arrayReader.getAsString()); + assertEquals(Lists.newArrayList(0L, 20L, 30L), arrayReader.getObject()); assertFalse(reader.next()); rs.clear(); @@ -373,7 +466,7 @@ public class TestScalarAccessors extends SubOperatorTest { assertTrue(reader.next()); assertEquals(Float.MAX_VALUE, colReader.getDouble(), 0.000001); - assertEquals((double) Float.MAX_VALUE, (double) colReader.getObject(), 0.000001); + assertEquals(Float.MAX_VALUE, (double) colReader.getObject(), 0.000001); assertTrue(reader.next()); assertEquals(Float.MIN_VALUE, colReader.getDouble(), 0.000001); @@ -433,23 +526,34 @@ public class TestScalarAccessors extends SubOperatorTest { assertEquals(2, rs.rowCount()); RowSetReader reader = rs.reader(); - ScalarElementReader colReader = reader.elements(0); + ArrayReader arrayReader = reader.array(0); + ScalarReader colReader = arrayReader.scalar(); assertEquals(ValueType.DOUBLE, colReader.valueType()); assertTrue(reader.next()); - assertEquals(0, colReader.size()); + assertEquals(0, arrayReader.size()); assertTrue(reader.next()); - assertEquals(3, colReader.size()); - assertEquals(0, colReader.getDouble(0), 0.00001); - assertEquals(20.5, colReader.getDouble(1), 0.00001); - assertEquals(30.0, colReader.getDouble(2), 0.00001); - assertEquals(0, (double) colReader.getObject(0), 0.00001); - assertEquals(20.5, (double) colReader.getObject(1), 0.00001); - assertEquals(30.0, (double) colReader.getObject(2), 0.00001); - assertEquals("0.0", colReader.getAsString(0)); - assertEquals("20.5", colReader.getAsString(1)); - assertEquals("30.0", colReader.getAsString(2)); + assertEquals(3, arrayReader.size()); + + assertTrue(arrayReader.next()); + assertEquals(0, colReader.getDouble(), 0.00001); + assertEquals(0, (double) colReader.getObject(), 0.00001); + assertEquals("0.0", colReader.getAsString()); + + assertTrue(arrayReader.next()); + assertEquals(20.5, colReader.getDouble(), 0.00001); + assertEquals(20.5, (double) colReader.getObject(), 0.00001); + assertEquals("20.5", colReader.getAsString()); + + assertTrue(arrayReader.next()); + assertEquals(30.0, colReader.getDouble(), 0.00001); + assertEquals(30.0, (double) colReader.getObject(), 0.00001); + assertEquals("30.0", colReader.getAsString()); + assertFalse(arrayReader.next()); + + assertEquals("[0.0, 20.5, 30.0]", arrayReader.getAsString()); + assertEquals(Lists.newArrayList(0.0D, 20.5D, 30D), arrayReader.getObject()); assertFalse(reader.next()); rs.clear(); @@ -507,15 +611,16 @@ public class TestScalarAccessors extends SubOperatorTest { } @Test - public void testStringRW() { + public void testVarcharRW() { BatchSchema batchSchema = new SchemaBuilder() .add("col", MinorType.VARCHAR) .build(); SingleRowSet rs = fixture.rowSetBuilder(batchSchema) .addRow("") - .addRow("abcd") + .addRow("fred") + .addRow("barney") .build(); - assertEquals(2, rs.rowCount()); + assertEquals(3, rs.rowCount()); RowSetReader reader = rs.reader(); ScalarReader colReader = reader.scalar(0); @@ -526,16 +631,21 @@ public class TestScalarAccessors extends SubOperatorTest { assertEquals("", colReader.getString()); assertTrue(reader.next()); - assertEquals("abcd", colReader.getString()); - assertEquals("abcd", colReader.getObject()); - assertEquals("\"abcd\"", colReader.getAsString()); + assertEquals("fred", colReader.getString()); + assertEquals("fred", colReader.getObject()); + assertEquals("\"fred\"", colReader.getAsString()); + + assertTrue(reader.next()); + assertEquals("barney", colReader.getString()); + assertEquals("barney", colReader.getObject()); + assertEquals("\"barney\"", colReader.getAsString()); assertFalse(reader.next()); rs.clear(); } @Test - public void testNullableString() { + public void testNullableVarchar() { BatchSchema batchSchema = new SchemaBuilder() .addNullable("col", MinorType.VARCHAR) .build(); @@ -568,7 +678,7 @@ public class TestScalarAccessors extends SubOperatorTest { } @Test - public void testStringArray() { + public void testVarcharArray() { BatchSchema batchSchema = new SchemaBuilder() .addArray("col", MinorType.VARCHAR) .build(); @@ -579,28 +689,71 @@ public class TestScalarAccessors extends SubOperatorTest { assertEquals(2, rs.rowCount()); RowSetReader reader = rs.reader(); - ScalarElementReader colReader = reader.elements(0); + ArrayReader arrayReader = reader.array(0); + ScalarReader colReader = arrayReader.scalar(); assertEquals(ValueType.STRING, colReader.valueType()); assertTrue(reader.next()); - assertEquals(0, colReader.size()); + assertEquals(0, arrayReader.size()); assertTrue(reader.next()); - assertEquals(3, colReader.size()); - assertEquals("fred", colReader.getString(0)); - assertEquals("", colReader.getString(1)); - assertEquals("wilma", colReader.getString(2)); - assertEquals("fred", colReader.getObject(0)); - assertEquals("", colReader.getObject(1)); - assertEquals("wilma", colReader.getObject(2)); - assertEquals("\"fred\"", colReader.getAsString(0)); - assertEquals("\"\"", colReader.getAsString(1)); - assertEquals("\"wilma\"", colReader.getAsString(2)); + assertEquals(3, arrayReader.size()); + + assertTrue(arrayReader.next()); + assertEquals("fred", colReader.getString()); + assertEquals("fred", colReader.getObject()); + assertEquals("\"fred\"", colReader.getAsString()); + + assertTrue(arrayReader.next()); + assertEquals("", colReader.getString()); + assertEquals("", colReader.getObject()); + assertEquals("\"\"", colReader.getAsString()); + + assertTrue(arrayReader.next()); + assertEquals("wilma", colReader.getString()); + assertEquals("wilma", colReader.getObject()); + assertEquals("\"wilma\"", colReader.getAsString()); + + assertFalse(arrayReader.next()); + + assertEquals("[\"fred\", \"\", \"wilma\"]", arrayReader.getAsString()); + assertEquals(Lists.newArrayList("fred", "", "wilma"), arrayReader.getObject()); assertFalse(reader.next()); rs.clear(); } + /** + * Test the low-level interval-year utilities used by the column accessors. + */ + + @Test + public void testIntervalYearUtils() { + { + Period expected = Period.months(0); + Period actual = DateUtilities.fromIntervalYear(0); + assertEquals(expected, actual.normalizedStandard()); + String fmt = DateUtilities.intervalYearStringBuilder(expected).toString(); + assertEquals("0 years 0 months", fmt); + } + + { + Period expected = Period.years(1).plusMonths(2); + Period actual = DateUtilities.fromIntervalYear(DateUtilities.periodToMonths(expected)); + assertEquals(expected, actual.normalizedStandard()); + String fmt = DateUtilities.intervalYearStringBuilder(expected).toString(); + assertEquals("1 year 2 months", fmt); + } + + { + Period expected = Period.years(6).plusMonths(1); + Period actual = DateUtilities.fromIntervalYear(DateUtilities.periodToMonths(expected)); + assertEquals(expected, actual.normalizedStandard()); + String fmt = DateUtilities.intervalYearStringBuilder(expected).toString(); + assertEquals("6 years 1 month", fmt); + } + } + @Test public void testIntervalYearRW() { BatchSchema batchSchema = new SchemaBuilder() @@ -664,7 +817,6 @@ public class TestScalarAccessors extends SubOperatorTest { assertTrue(reader.next()); assertTrue(colReader.isNull()); - assertNull(colReader.getPeriod()); assertNull(colReader.getObject()); assertEquals("null", colReader.getAsString()); @@ -692,24 +844,64 @@ public class TestScalarAccessors extends SubOperatorTest { assertEquals(2, rs.rowCount()); RowSetReader reader = rs.reader(); - ScalarElementReader colReader = reader.elements(0); + ArrayReader arrayReader = reader.array(0); + ScalarReader colReader = arrayReader.scalar(); assertEquals(ValueType.PERIOD, colReader.valueType()); assertTrue(reader.next()); - assertEquals(0, colReader.size()); + assertEquals(0, arrayReader.size()); assertTrue(reader.next()); - assertEquals(3, colReader.size()); - assertEquals(p1, colReader.getPeriod(0)); - assertEquals(p2, colReader.getPeriod(1)); - assertEquals(p3, colReader.getPeriod(2)); - assertEquals(p2, colReader.getObject(1)); - assertEquals(p2.toString(), colReader.getAsString(1)); + assertEquals(3, arrayReader.size()); + + assertTrue(arrayReader.next()); + assertEquals(p1, colReader.getPeriod()); + + assertTrue(arrayReader.next()); + assertEquals(p2, colReader.getPeriod()); + assertEquals(p2, colReader.getObject()); + assertEquals(p2.toString(), colReader.getAsString()); + + assertTrue(arrayReader.next()); + assertEquals(p3, colReader.getPeriod()); + + assertFalse(arrayReader.next()); assertFalse(reader.next()); rs.clear(); } + /** + * Test the low-level interval-day utilities used by the column accessors. + */ + + @Test + public void testIntervalDayUtils() { + { + Period expected = Period.days(0); + Period actual = DateUtilities.fromIntervalDay(0, 0); + assertEquals(expected, actual.normalizedStandard()); + String fmt = DateUtilities.intervalDayStringBuilder(expected).toString(); + assertEquals("0 days 0:00:00", fmt); + } + + { + Period expected = Period.days(1).plusHours(5).plusMinutes(6).plusSeconds(7); + Period actual = DateUtilities.fromIntervalDay(1, DateUtilities.timeToMillis(5, 6, 7, 0)); + assertEquals(expected, actual.normalizedStandard()); + String fmt = DateUtilities.intervalDayStringBuilder(expected).toString(); + assertEquals("1 day 5:06:07", fmt); + } + + { + Period expected = Period.days(2).plusHours(12).plusMinutes(23).plusSeconds(34).plusMillis(567); + Period actual = DateUtilities.fromIntervalDay(2, DateUtilities.timeToMillis(12, 23, 34, 567)); + assertEquals(expected, actual.normalizedStandard()); + String fmt = DateUtilities.intervalDayStringBuilder(expected).toString(); + assertEquals("2 days 12:23:34.567", fmt); + } + } + @Test public void testIntervalDayRW() { BatchSchema batchSchema = new SchemaBuilder() @@ -774,7 +966,6 @@ public class TestScalarAccessors extends SubOperatorTest { assertTrue(reader.next()); assertTrue(colReader.isNull()); - assertNull(colReader.getPeriod()); assertNull(colReader.getObject()); assertEquals("null", colReader.getAsString()); @@ -802,24 +993,79 @@ public class TestScalarAccessors extends SubOperatorTest { assertEquals(2, rs.rowCount()); RowSetReader reader = rs.reader(); - ScalarElementReader colReader = reader.elements(0); + ArrayReader arrayReader = reader.array(0); + ScalarReader colReader = arrayReader.scalar(); assertEquals(ValueType.PERIOD, colReader.valueType()); assertTrue(reader.next()); - assertEquals(0, colReader.size()); + assertEquals(0, arrayReader.size()); assertTrue(reader.next()); - assertEquals(3, colReader.size()); - assertEquals(p1, colReader.getPeriod(0).normalizedStandard()); - assertEquals(p2, colReader.getPeriod(1).normalizedStandard()); - assertEquals(p3.normalizedStandard(), colReader.getPeriod(2).normalizedStandard()); - assertEquals(p2, ((Period) colReader.getObject(1)).normalizedStandard()); - assertEquals(p2.toString(), colReader.getAsString(1)); + assertEquals(3, arrayReader.size()); + + assertTrue(arrayReader.next()); + assertEquals(p1, colReader.getPeriod().normalizedStandard()); + + assertTrue(arrayReader.next()); + assertEquals(p2, colReader.getPeriod().normalizedStandard()); + assertEquals(p2, ((Period) colReader.getObject()).normalizedStandard()); + assertEquals(p2.toString(), colReader.getAsString()); + + assertTrue(arrayReader.next()); + assertEquals(p3.normalizedStandard(), colReader.getPeriod().normalizedStandard()); + + assertFalse(arrayReader.next()); assertFalse(reader.next()); rs.clear(); } + /** + * Test the low-level interval utilities used by the column accessors. + */ + + @Test + public void testIntervalUtils() { + { + Period expected = Period.months(0); + Period actual = DateUtilities.fromInterval(0, 0, 0); + assertEquals(expected, actual.normalizedStandard()); + String fmt = DateUtilities.intervalStringBuilder(expected).toString(); + assertEquals("0 years 0 months 0 days 0:00:00", fmt); + } + + { + Period expected = Period.years(1).plusMonths(2).plusDays(3) + .plusHours(5).plusMinutes(6).plusSeconds(7); + Period actual = DateUtilities.fromInterval(DateUtilities.periodToMonths(expected), 3, + DateUtilities.periodToMillis(expected)); + assertEquals(expected, actual.normalizedStandard()); + String fmt = DateUtilities.intervalStringBuilder(expected).toString(); + assertEquals("1 year 2 months 3 days 5:06:07", fmt); + } + + { + Period expected = Period.years(2).plusMonths(1).plusDays(3) + .plusHours(12).plusMinutes(23).plusSeconds(34) + .plusMillis(456); + Period actual = DateUtilities.fromInterval(DateUtilities.periodToMonths(expected), 3, + DateUtilities.periodToMillis(expected)); + assertEquals(expected, actual.normalizedStandard()); + String fmt = DateUtilities.intervalStringBuilder(expected).toString(); + assertEquals("2 years 1 month 3 days 12:23:34.456", fmt); + } + + { + Period expected = Period.years(2).plusMonths(3).plusDays(1) + .plusHours(12).plusMinutes(23).plusSeconds(34); + Period actual = DateUtilities.fromInterval(DateUtilities.periodToMonths(expected), 1, + DateUtilities.periodToMillis(expected)); + assertEquals(expected, actual.normalizedStandard()); + String fmt = DateUtilities.intervalStringBuilder(expected).toString(); + assertEquals("2 years 3 months 1 day 12:23:34", fmt); + } + } + @Test public void testIntervalRW() { BatchSchema batchSchema = new SchemaBuilder() @@ -890,7 +1136,6 @@ public class TestScalarAccessors extends SubOperatorTest { assertTrue(reader.next()); assertTrue(colReader.isNull()); - assertNull(colReader.getPeriod()); assertNull(colReader.getObject()); assertEquals("null", colReader.getAsString()); @@ -922,19 +1167,28 @@ public class TestScalarAccessors extends SubOperatorTest { assertEquals(2, rs.rowCount()); RowSetReader reader = rs.reader(); - ScalarElementReader colReader = reader.elements(0); + ArrayReader arrayReader = reader.array(0); + ScalarReader colReader = arrayReader.scalar(); assertEquals(ValueType.PERIOD, colReader.valueType()); assertTrue(reader.next()); - assertEquals(0, colReader.size()); + assertEquals(0, arrayReader.size()); assertTrue(reader.next()); - assertEquals(3, colReader.size()); - assertEquals(p1, colReader.getPeriod(0).normalizedStandard()); - assertEquals(p2, colReader.getPeriod(1).normalizedStandard()); - assertEquals(p3.normalizedStandard(), colReader.getPeriod(2).normalizedStandard()); - assertEquals(p2, ((Period) colReader.getObject(1)).normalizedStandard()); - assertEquals(p2.toString(), colReader.getAsString(1)); + assertEquals(3, arrayReader.size()); + + assertTrue(arrayReader.next()); + assertEquals(p1, colReader.getPeriod().normalizedStandard()); + + assertTrue(arrayReader.next()); + assertEquals(p2, colReader.getPeriod().normalizedStandard()); + assertEquals(p2, ((Period) colReader.getObject()).normalizedStandard()); + assertEquals(p2.toString(), colReader.getAsString()); + + assertTrue(arrayReader.next()); + assertEquals(p3.normalizedStandard(), colReader.getPeriod().normalizedStandard()); + + assertFalse(arrayReader.next()); assertFalse(reader.next()); rs.clear(); @@ -1051,19 +1305,28 @@ public class TestScalarAccessors extends SubOperatorTest { assertEquals(2, rs.rowCount()); RowSetReader reader = rs.reader(); - ScalarElementReader colReader = reader.elements(0); + ArrayReader arrayReader = reader.array(0); + ScalarReader colReader = arrayReader.scalar(); assertEquals(ValueType.DECIMAL, colReader.valueType()); assertTrue(reader.next()); - assertEquals(0, colReader.size()); + assertEquals(0, arrayReader.size()); assertTrue(reader.next()); - assertEquals(3, colReader.size()); - assertEquals(0, v1.compareTo(colReader.getDecimal(0))); - assertEquals(0, v2.compareTo(colReader.getDecimal(1))); - assertEquals(0, v3.compareTo(colReader.getDecimal(2))); - assertEquals(0, v2.compareTo((BigDecimal) colReader.getObject(1))); - assertEquals(v2.toString(), colReader.getAsString(1)); + assertEquals(3, arrayReader.size()); + + assertTrue(arrayReader.next()); + assertEquals(0, v1.compareTo(colReader.getDecimal())); + + assertTrue(arrayReader.next()); + assertEquals(0, v2.compareTo(colReader.getDecimal())); + assertEquals(0, v2.compareTo((BigDecimal) colReader.getObject())); + assertEquals(v2.toString(), colReader.getAsString()); + + assertTrue(arrayReader.next()); + assertEquals(0, v3.compareTo(colReader.getDecimal())); + + assertFalse(arrayReader.next()); assertFalse(reader.next()); rs.clear(); @@ -1246,19 +1509,28 @@ public class TestScalarAccessors extends SubOperatorTest { assertEquals(2, rs.rowCount()); RowSetReader reader = rs.reader(); - ScalarElementReader colReader = reader.elements(0); + ArrayReader arrayReader = reader.array(0); + ScalarReader colReader = arrayReader.scalar(); assertEquals(ValueType.BYTES, colReader.valueType()); assertTrue(reader.next()); - assertEquals(0, colReader.size()); + assertEquals(0, arrayReader.size()); assertTrue(reader.next()); - assertEquals(3, colReader.size()); - assertTrue(Arrays.equals(v1, colReader.getBytes(0))); - assertTrue(Arrays.equals(v2, colReader.getBytes(1))); - assertTrue(Arrays.equals(v3, colReader.getBytes(2))); - assertTrue(Arrays.equals(v2, (byte[]) colReader.getObject(1))); - assertEquals("[00, 7f, 80, ff]", colReader.getAsString(1)); + assertEquals(3, arrayReader.size()); + + assertTrue(arrayReader.next()); + assertTrue(Arrays.equals(v1, colReader.getBytes())); + + assertTrue(arrayReader.next()); + assertTrue(Arrays.equals(v2, colReader.getBytes())); + assertTrue(Arrays.equals(v2, (byte[]) colReader.getObject())); + assertEquals("[00, 7f, 80, ff]", colReader.getAsString()); + + assertTrue(arrayReader.next()); + assertTrue(Arrays.equals(v3, colReader.getBytes())); + + assertFalse(arrayReader.next()); assertFalse(reader.next()); rs.clear();
