http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java index e106171..093c4ae 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java @@ -32,7 +32,7 @@ import org.apache.drill.exec.physical.config.Sort; import org.apache.drill.exec.physical.impl.xsort.managed.PriorityQueueCopierWrapper.BatchMerger; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; -import org.apache.drill.exec.record.TupleMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.record.VectorContainer; import org.apache.drill.test.OperatorFixture; import org.apache.drill.test.rowSet.DirectRowSet;
http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMapArray.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMapArray.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMapArray.java index 115e52d..86e69aa 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMapArray.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMapArray.java @@ -27,7 +27,7 @@ import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.physical.rowSet.ResultSetLoader; import org.apache.drill.exec.physical.rowSet.RowSetLoader; -import org.apache.drill.exec.record.TupleMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.ArrayReader; import org.apache.drill.exec.vector.accessor.ArrayWriter; http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMaps.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMaps.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMaps.java index b23eb0d..2aaa934 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMaps.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMaps.java @@ -29,7 +29,7 @@ import org.apache.drill.exec.physical.rowSet.ResultSetLoader; import org.apache.drill.exec.physical.rowSet.RowSetLoader; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; -import org.apache.drill.exec.record.TupleMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.apache.drill.exec.vector.accessor.TupleReader; http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOmittedValues.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOmittedValues.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOmittedValues.java index 2c4c87b..b0df724 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOmittedValues.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOmittedValues.java @@ -26,7 +26,7 @@ import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.physical.rowSet.ResultSetLoader; import org.apache.drill.exec.physical.rowSet.RowSetLoader; import org.apache.drill.exec.record.BatchSchema; -import org.apache.drill.exec.record.TupleMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.apache.drill.test.SubOperatorTest; http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOverflow.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOverflow.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOverflow.java index 0146cfe..c9ab4e0 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOverflow.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOverflow.java @@ -30,7 +30,7 @@ import org.apache.drill.exec.physical.rowSet.ResultSetLoader; import org.apache.drill.exec.physical.rowSet.RowSetLoader; import org.apache.drill.exec.physical.rowSet.impl.ResultSetLoaderImpl.ResultSetOptions; import org.apache.drill.exec.record.MaterializedField; -import org.apache.drill.exec.record.TupleMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.ScalarElementReader; import org.apache.drill.exec.vector.accessor.ScalarWriter; http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java index 5c6ff7b..e28745b 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java @@ -30,8 +30,8 @@ import org.apache.drill.exec.physical.rowSet.ResultSetLoader; import org.apache.drill.exec.physical.rowSet.RowSetLoader; import org.apache.drill.exec.physical.rowSet.impl.ResultSetLoaderImpl.ResultSetOptions; import org.apache.drill.exec.record.BatchSchema; -import org.apache.drill.exec.record.ColumnMetadata; -import org.apache.drill.exec.record.TupleMetadata; +import org.apache.drill.exec.record.metadata.ColumnMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.RowSet; http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java index ffcc84a..1c4082b 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java @@ -32,7 +32,8 @@ import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.physical.rowSet.ResultSetLoader; import org.apache.drill.exec.physical.rowSet.RowSetLoader; import org.apache.drill.exec.record.MaterializedField; -import org.apache.drill.exec.record.TupleMetadata; +import org.apache.drill.exec.record.metadata.ColumnMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.apache.drill.exec.vector.accessor.TupleWriter.UndefinedColumnException; @@ -98,8 +99,8 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { rootWriter.addColumn(fieldA); assertEquals(1, schema.size()); - assertSame(fieldA, schema.column(0)); - assertSame(fieldA, schema.column("a")); + assertTrue(fieldA.isEquivalent(schema.column(0))); + assertSame(schema.metadata(0), schema.metadata("a")); // Error to start a row before the first batch. @@ -148,8 +149,8 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { rootWriter.addColumn(fieldB); assertEquals(2, schema.size()); - assertSame(fieldB, schema.column(1)); - assertSame(fieldB, schema.column("b")); + assertTrue(fieldB.isEquivalent(schema.column(1))); + assertSame(schema.metadata(1), schema.metadata("b")); rootWriter.start(); rootWriter.scalar(0).setInt(200); @@ -298,6 +299,12 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { * Schemas are case insensitive by default. Verify that * the schema mechanism works, with emphasis on the * case insensitive case. + * <p> + * The tests here and elsewhere build columns from a + * <tt>MaterializedField</tt>. Doing so is rather old-school; + * better to use the newer <tt>ColumnMetadata</tt> which provides + * additional information. The code here simply uses the <tt>MaterializedField</tt> + * to create a <tt>ColumnMetadata</tt> implicitly. */ @Test @@ -305,10 +312,11 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator()); RowSetLoader rootWriter = rsLoader.writer(); TupleMetadata schema = rootWriter.schema(); + assertEquals(0, rsLoader.schemaVersion()); // No columns defined in schema - assertNull(schema.column("a")); + assertNull(schema.metadata("a")); try { schema.column(0); fail(); @@ -333,14 +341,17 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { // Define a column + assertEquals(0, rsLoader.schemaVersion()); MaterializedField colSchema = SchemaBuilder.columnSchema("a", MinorType.VARCHAR, DataMode.REQUIRED); rootWriter.addColumn(colSchema); + assertEquals(1, rsLoader.schemaVersion()); // Can now be found, case insensitive - assertSame(colSchema, schema.column(0)); - assertSame(colSchema, schema.column("a")); - assertSame(colSchema, schema.column("A")); + assertTrue(colSchema.isEquivalent(schema.column(0))); + ColumnMetadata colMetadata = schema.metadata(0); + assertSame(colMetadata, schema.metadata("a")); + assertSame(colMetadata, schema.metadata("A")); assertNotNull(rootWriter.column(0)); assertNotNull(rootWriter.column("a")); assertNotNull(rootWriter.column("A")); @@ -373,9 +384,10 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { MaterializedField col2 = SchemaBuilder.columnSchema("b", MinorType.VARCHAR, DataMode.REQUIRED); rootWriter.addColumn(col2); - assertSame(col2, schema.column(1)); - assertSame(col2, schema.column("b")); - assertSame(col2, schema.column("B")); + assertTrue(col2.isEquivalent(schema.column(1))); + ColumnMetadata col2Metadata = schema.metadata(1); + assertSame(col2Metadata, schema.metadata("b")); + assertSame(col2Metadata, schema.metadata("B")); assertEquals(2, schema.size()); assertEquals(1, schema.index("b")); assertEquals(1, schema.index("B")); @@ -391,9 +403,10 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { MaterializedField col3 = SchemaBuilder.columnSchema("c", MinorType.VARCHAR, DataMode.REQUIRED); rootWriter.addColumn(col3); - assertSame(col3, schema.column(2)); - assertSame(col3, schema.column("c")); - assertSame(col3, schema.column("C")); + assertTrue(col3.isEquivalent(schema.column(2))); + ColumnMetadata col3Metadata = schema.metadata(2); + assertSame(col3Metadata, schema.metadata("c")); + assertSame(col3Metadata, schema.metadata("C")); assertEquals(3, schema.size()); assertEquals(2, schema.index("c")); assertEquals(2, schema.index("C")); @@ -401,9 +414,10 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { MaterializedField col4 = SchemaBuilder.columnSchema("d", MinorType.VARCHAR, DataMode.OPTIONAL); rootWriter.addColumn(col4); - assertSame(col4, schema.column(3)); - assertSame(col4, schema.column("d")); - assertSame(col4, schema.column("D")); + assertTrue(col4.isEquivalent(schema.column(3))); + ColumnMetadata col4Metadata = schema.metadata(3); + assertSame(col4Metadata, schema.metadata("d")); + assertSame(col4Metadata, schema.metadata("D")); assertEquals(4, schema.size()); assertEquals(3, schema.index("d")); assertEquals(3, schema.index("D")); @@ -411,9 +425,10 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { MaterializedField col5 = SchemaBuilder.columnSchema("e", MinorType.VARCHAR, DataMode.REPEATED); rootWriter.addColumn(col5); - assertSame(col5, schema.column(4)); - assertSame(col5, schema.column("e")); - assertSame(col5, schema.column("E")); + assertTrue(col5.isEquivalent(schema.column(4))); + ColumnMetadata col5Metadata = schema.metadata(4); + assertSame(col5Metadata, schema.metadata("e")); + assertSame(col5Metadata, schema.metadata("E")); assertEquals(5, schema.size()); assertEquals(4, schema.index("e")); assertEquals(4, schema.index("E")); http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java index 33b9826..22a42f6 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java @@ -25,7 +25,7 @@ import java.util.Arrays; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.physical.rowSet.ResultSetLoader; import org.apache.drill.exec.physical.rowSet.RowSetLoader; -import org.apache.drill.exec.record.TupleMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.ArrayReader; import org.apache.drill.exec.vector.accessor.ArrayWriter; http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestMiniPlan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestMiniPlan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestMiniPlan.java index 79b9dbc..e6ddc87 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestMiniPlan.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestMiniPlan.java @@ -18,10 +18,13 @@ package org.apache.drill.exec.physical.unit; -import com.google.common.collect.Lists; +import java.util.Collections; +import java.util.List; + import org.apache.drill.categories.PlannerTest; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.util.DrillFileUtils; +import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.physical.config.Filter; import org.apache.drill.exec.physical.config.UnionAll; import org.apache.drill.exec.record.BatchSchema; @@ -34,8 +37,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; -import java.util.Collections; -import java.util.List; +import com.google.common.collect.Lists; /** * This class contains examples to show how to use MiniPlanTestBuilder to test a @@ -110,7 +112,7 @@ public class TestMiniPlan extends MiniPlanUnitTestBase { "[{\"a\": 50, \"b\" : 100}]"); RecordBatch batch = new PopBuilder() - .physicalOperator(new UnionAll(Collections.EMPTY_LIST)) // Children list is provided through RecordBatch + .physicalOperator(new UnionAll(Collections.<PhysicalOperator> emptyList())) // Children list is provided through RecordBatch .addInputAsChild() .physicalOperator(new Filter(null, parseExpr("a=5"), 1.0f)) .addJsonScanAsChild() http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestTupleSchema.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestTupleSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestTupleSchema.java index 45c0b55..83bdcaf 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestTupleSchema.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestTupleSchema.java @@ -29,12 +29,21 @@ import java.util.Iterator; import java.util.List; import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.common.types.Types; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; -import org.apache.drill.exec.record.TupleSchema.MapColumnMetadata; -import org.apache.drill.exec.record.TupleSchema.PrimitiveColumnMetadata; +import org.apache.drill.exec.record.metadata.ColumnMetadata; +import org.apache.drill.exec.record.metadata.MapColumnMetadata; +import org.apache.drill.exec.record.metadata.MetadataUtils; +import org.apache.drill.exec.record.metadata.PrimitiveColumnMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.exec.record.metadata.TupleSchema; +import org.apache.drill.exec.record.metadata.VariantColumnMetadata; +import org.apache.drill.exec.record.metadata.VariantMetadata; import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.SchemaBuilder; +import org.apache.drill.test.rowSet.SchemaBuilder.ColumnBuilder; import org.junit.Test; /** @@ -53,7 +62,7 @@ public class TestTupleSchema extends SubOperatorTest { public void testRequiredFixedWidthColumn() { MaterializedField field = SchemaBuilder.columnSchema("c", MinorType.INT, DataMode.REQUIRED ); - ColumnMetadata col = TupleSchema.fromField(field); + ColumnMetadata col = MetadataUtils.fromField(field); // Code may depend on the specific column class @@ -63,31 +72,30 @@ public class TestTupleSchema extends SubOperatorTest { assertEquals(ColumnMetadata.StructureType.PRIMITIVE, col.structureType()); assertNull(col.mapSchema()); - assertSame(field, col.schema()); + assertTrue(field.isEquivalent(col.schema())); assertEquals(field.getName(), col.name()); - assertEquals(field.getType(), col.majorType()); assertEquals(field.getType().getMinorType(), col.type()); assertEquals(field.getDataMode(), col.mode()); assertFalse(col.isNullable()); assertFalse(col.isArray()); assertFalse(col.isVariableWidth()); assertFalse(col.isMap()); - assertFalse(col.isList()); assertTrue(col.isEquivalent(col)); + assertFalse(col.isVariant()); - ColumnMetadata col2 = TupleSchema.fromField(field); + ColumnMetadata col2 = MetadataUtils.fromField(field); assertTrue(col.isEquivalent(col2)); MaterializedField field3 = SchemaBuilder.columnSchema("d", MinorType.INT, DataMode.REQUIRED ); - ColumnMetadata col3 = TupleSchema.fromField(field3); + ColumnMetadata col3 = MetadataUtils.fromField(field3); assertFalse(col.isEquivalent(col3)); MaterializedField field4 = SchemaBuilder.columnSchema("c", MinorType.BIGINT, DataMode.REQUIRED ); - ColumnMetadata col4 = TupleSchema.fromField(field4); + ColumnMetadata col4 = MetadataUtils.fromField(field4); assertFalse(col.isEquivalent(col4)); MaterializedField field5 = SchemaBuilder.columnSchema("c", MinorType.INT, DataMode.OPTIONAL ); - ColumnMetadata col5 = TupleSchema.fromField(field5); + ColumnMetadata col5 = MetadataUtils.fromField(field5); assertFalse(col.isEquivalent(col5)); ColumnMetadata col6 = col.cloneEmpty(); @@ -106,14 +114,14 @@ public class TestTupleSchema extends SubOperatorTest { public void testNullableFixedWidthColumn() { MaterializedField field = SchemaBuilder.columnSchema("c", MinorType.INT, DataMode.OPTIONAL ); - ColumnMetadata col = TupleSchema.fromField(field); + ColumnMetadata col = MetadataUtils.fromField(field); assertEquals(ColumnMetadata.StructureType.PRIMITIVE, col.structureType()); assertTrue(col.isNullable()); assertFalse(col.isArray()); assertFalse(col.isVariableWidth()); assertFalse(col.isMap()); - assertFalse(col.isList()); + assertFalse(col.isVariant()); assertEquals(4, col.expectedWidth()); col.setExpectedWidth(10); @@ -128,13 +136,13 @@ public class TestTupleSchema extends SubOperatorTest { public void testRepeatedFixedWidthColumn() { MaterializedField field = SchemaBuilder.columnSchema("c", MinorType.INT, DataMode.REPEATED ); - ColumnMetadata col = TupleSchema.fromField(field); + ColumnMetadata col = MetadataUtils.fromField(field); assertFalse(col.isNullable()); assertTrue(col.isArray()); assertFalse(col.isVariableWidth()); assertFalse(col.isMap()); - assertFalse(col.isList()); + assertFalse(col.isVariant()); assertEquals(4, col.expectedWidth()); col.setExpectedWidth(10); @@ -153,7 +161,7 @@ public class TestTupleSchema extends SubOperatorTest { public void testRequiredVariableWidthColumn() { MaterializedField field = SchemaBuilder.columnSchema("c", MinorType.VARCHAR, DataMode.REQUIRED ); - ColumnMetadata col = TupleSchema.fromField(field); + ColumnMetadata col = MetadataUtils.fromField(field); assertEquals(ColumnMetadata.StructureType.PRIMITIVE, col.structureType()); assertNull(col.mapSchema()); @@ -161,16 +169,16 @@ public class TestTupleSchema extends SubOperatorTest { assertFalse(col.isArray()); assertTrue(col.isVariableWidth()); assertFalse(col.isMap()); - assertFalse(col.isList()); + assertFalse(col.isVariant()); // A different precision is a different type. - MaterializedField field2 = new SchemaBuilder.ColumnBuilder("c", MinorType.VARCHAR) + MaterializedField field2 = new ColumnBuilder("c", MinorType.VARCHAR) .setMode(DataMode.REQUIRED) .setPrecision(10) .build(); - ColumnMetadata col2 = TupleSchema.fromField(field2); + ColumnMetadata col2 = MetadataUtils.fromField(field2); assertFalse(col.isEquivalent(col2)); assertEquals(50, col.expectedWidth()); @@ -183,7 +191,7 @@ public class TestTupleSchema extends SubOperatorTest { // If precision is provided, then that is the default width - col = TupleSchema.fromField(field2); + col = MetadataUtils.fromField(field2); assertEquals(10, col.expectedWidth()); } @@ -191,13 +199,13 @@ public class TestTupleSchema extends SubOperatorTest { public void testNullableVariableWidthColumn() { MaterializedField field = SchemaBuilder.columnSchema("c", MinorType.VARCHAR, DataMode.OPTIONAL ); - ColumnMetadata col = TupleSchema.fromField(field); + ColumnMetadata col = MetadataUtils.fromField(field); assertTrue(col.isNullable()); assertFalse(col.isArray()); assertTrue(col.isVariableWidth()); assertFalse(col.isMap()); - assertFalse(col.isList()); + assertFalse(col.isVariant()); assertEquals(50, col.expectedWidth()); col.setExpectedWidth(10); @@ -212,13 +220,13 @@ public class TestTupleSchema extends SubOperatorTest { public void testRepeatedVariableWidthColumn() { MaterializedField field = SchemaBuilder.columnSchema("c", MinorType.VARCHAR, DataMode.REPEATED ); - ColumnMetadata col = TupleSchema.fromField(field); + ColumnMetadata col = MetadataUtils.fromField(field); assertFalse(col.isNullable()); assertTrue(col.isArray()); assertTrue(col.isVariableWidth()); assertFalse(col.isMap()); - assertFalse(col.isList()); + assertFalse(col.isVariant()); assertEquals(50, col.expectedWidth()); col.setExpectedWidth(10); @@ -230,6 +238,31 @@ public class TestTupleSchema extends SubOperatorTest { assertEquals(2, col.expectedElementCount()); } + @Test + public void testDecimalScalePrecision() { + + MaterializedField field = MaterializedField.create("d", + MajorType.newBuilder() + .setMinorType(MinorType.DECIMAL9) + .setMode(DataMode.REQUIRED) + .setPrecision(3) + .setScale(4) + .build()); + + ColumnMetadata col = MetadataUtils.fromField(field); + + assertFalse(col.isNullable()); + assertFalse(col.isArray()); + assertFalse(col.isVariableWidth()); + assertFalse(col.isMap()); + assertFalse(col.isVariant()); + + assertEquals(3, col.precision()); + assertEquals(4, col.scale()); + + assertTrue(field.isEquivalent(col.schema())); + } + /** * Tests a map column. Maps can only be required or repeated, not nullable. * (But, the columns in the map can be nullable.) @@ -239,7 +272,7 @@ public class TestTupleSchema extends SubOperatorTest { public void testMapColumn() { MaterializedField field = SchemaBuilder.columnSchema("m", MinorType.MAP, DataMode.REQUIRED ); - ColumnMetadata col = TupleSchema.fromField(field); + ColumnMetadata col = MetadataUtils.fromField(field); assertTrue(col instanceof MapColumnMetadata); assertNotNull(col.mapSchema()); @@ -254,7 +287,7 @@ public class TestTupleSchema extends SubOperatorTest { assertFalse(col.isArray()); assertFalse(col.isVariableWidth()); assertTrue(col.isMap()); - assertFalse(col.isList()); + assertFalse(col.isVariant()); assertEquals(0, col.expectedWidth()); col.setExpectedWidth(10); @@ -269,7 +302,7 @@ public class TestTupleSchema extends SubOperatorTest { public void testRepeatedMapColumn() { MaterializedField field = SchemaBuilder.columnSchema("m", MinorType.MAP, DataMode.REPEATED ); - ColumnMetadata col = TupleSchema.fromField(field); + ColumnMetadata col = MetadataUtils.fromField(field); assertTrue(col instanceof MapColumnMetadata); assertNotNull(col.mapSchema()); @@ -279,7 +312,7 @@ public class TestTupleSchema extends SubOperatorTest { assertTrue(col.isArray()); assertFalse(col.isVariableWidth()); assertTrue(col.isMap()); - assertFalse(col.isList()); + assertFalse(col.isVariant()); assertEquals(0, col.expectedWidth()); col.setExpectedWidth(10); @@ -291,9 +324,75 @@ public class TestTupleSchema extends SubOperatorTest { assertEquals(2, col.expectedElementCount()); } - // List + @Test + public void testUnionColumn() { + + MaterializedField field = SchemaBuilder.columnSchema("u", MinorType.UNION, DataMode.OPTIONAL); + ColumnMetadata col = MetadataUtils.fromField(field); + assertFalse(col.isArray()); + doVariantTest(col); + } + + @Test + public void testListColumn() { + + MaterializedField field = SchemaBuilder.columnSchema("l", MinorType.LIST, DataMode.OPTIONAL); + ColumnMetadata col = MetadataUtils.fromField(field); + assertTrue(col.isArray()); + + // List modeled as a repeated element. Implementation is a bit + // more complex, but does not affect this abstract description. + + doVariantTest(col); + } + + private void doVariantTest(ColumnMetadata col) { - // Repeated list + assertTrue(col instanceof VariantColumnMetadata); + + assertTrue(col.isNullable()); + assertFalse(col.isVariableWidth()); + assertFalse(col.isMap()); + assertTrue(col.isVariant()); + + assertEquals(0, col.expectedWidth()); + col.setExpectedWidth(10); + assertEquals(0, col.expectedWidth()); + + VariantMetadata variant = col.variantSchema(); + assertNotNull(variant); + assertEquals(0, variant.size()); + + ColumnMetadata member = variant.addType(MinorType.INT); + assertEquals(MinorType.INT, member.type()); + assertEquals(DataMode.OPTIONAL, member.mode()); + assertEquals(Types.typeKey(MinorType.INT), member.name()); + + assertEquals(1, variant.size()); + assertTrue(variant.hasType(MinorType.INT)); + assertSame(member, variant.member(MinorType.INT)); + + try { + variant.addType(MinorType.INT); + fail(); + } catch (IllegalArgumentException e) { + // expected + } + + assertFalse(variant.hasType(MinorType.VARCHAR)); + member = variant.addType(MinorType.VARCHAR); + assertEquals(MinorType.VARCHAR, member.type()); + assertEquals(DataMode.OPTIONAL, member.mode()); + assertEquals(Types.typeKey(MinorType.VARCHAR), member.name()); + + assertEquals(2, variant.size()); + assertTrue(variant.hasType(MinorType.VARCHAR)); + assertSame(member, variant.member(MinorType.VARCHAR)); + + assertFalse(variant.hasType(MinorType.BIGINT)); + } + + // Repeated list /** * Test the basics of an empty root tuple (i.e. row) schema. @@ -335,7 +434,7 @@ public class TestTupleSchema extends SubOperatorTest { // in the tuple. MaterializedField field = SchemaBuilder.columnSchema("c", MinorType.INT, DataMode.REQUIRED ); - ColumnMetadata col = TupleSchema.fromField(field); + ColumnMetadata col = MetadataUtils.fromField(field); assertEquals("c", root.fullName(col)); assertTrue(root.isEquivalent(root)); @@ -361,9 +460,9 @@ public class TestTupleSchema extends SubOperatorTest { assertEquals(0, root.index("a")); assertEquals(-1, root.index("b")); - assertSame(fieldA, root.column(0)); - assertSame(fieldA, root.column("a")); - assertSame(fieldA, root.column("A")); + assertTrue(fieldA.isEquivalent(root.column(0))); + assertTrue(fieldA.isEquivalent(root.column("a"))); + assertTrue(fieldA.isEquivalent(root.column("A"))); assertSame(colA, root.metadata(0)); assertSame(colA, root.metadata("a")); @@ -379,7 +478,7 @@ public class TestTupleSchema extends SubOperatorTest { } MaterializedField fieldB = SchemaBuilder.columnSchema("b", MinorType.VARCHAR, DataMode.OPTIONAL ); - ColumnMetadata colB = TupleSchema.fromField(fieldB); + ColumnMetadata colB = MetadataUtils.fromField(fieldB); int indexB = root.addColumn(colB); assertEquals(1, indexB); @@ -387,8 +486,8 @@ public class TestTupleSchema extends SubOperatorTest { assertFalse(root.isEmpty()); assertEquals(indexB, root.index("b")); - assertSame(fieldB, root.column(1)); - assertSame(fieldB, root.column("b")); + assertTrue(fieldB.isEquivalent(root.column(1))); + assertTrue(fieldB.isEquivalent(root.column("b"))); assertSame(colB, root.metadata(1)); assertSame(colB, root.metadata("b")); @@ -404,8 +503,8 @@ public class TestTupleSchema extends SubOperatorTest { } List<MaterializedField> fieldList = root.toFieldList(); - assertSame(fieldA, fieldList.get(0)); - assertSame(fieldB, fieldList.get(1)); + assertTrue(fieldA.isEquivalent(fieldList.get(0))); + assertTrue(fieldB.isEquivalent(fieldList.get(1))); TupleMetadata emptyRoot = new TupleSchema(); assertFalse(emptyRoot.isEquivalent(root)); @@ -433,7 +532,7 @@ public class TestTupleSchema extends SubOperatorTest { // And it is equivalent to the round trip to a batch schema. BatchSchema batchSchema = ((TupleSchema) root).toBatchSchema(SelectionVectorMode.NONE); - assertTrue(root.isEquivalent(TupleSchema.fromFields(batchSchema))); + assertTrue(root.isEquivalent(MetadataUtils.fromFields(batchSchema))); } /** @@ -441,10 +540,12 @@ public class TestTupleSchema extends SubOperatorTest { * a.`b.x`.`c.y`.d<br> * in which columns "a", "b.x" and "c.y" are maps, "b.x" and "c.y" are names * that contains dots, and d is primitive. + * Here we build up the schema using the metadata schema, and generate a + * materialized field from the metadata. */ @Test - public void testMapTuple() { + public void testMapTupleFromMetadata() { TupleMetadata root = new TupleSchema(); @@ -492,18 +593,91 @@ public class TestTupleSchema extends SubOperatorTest { // Yes, it is awful that MaterializedField does not provide indexed // access to its children. That's one reason we have the TupleMetadata - // classes.. + // classes... + // Note that the metadata layer does not store the materialized field. + // (Doing so causes no end of synchronization problems.) So we test + // for equivalence, not sameness. - assertSame(fieldB, colA.schema().getChildren().iterator().next()); - assertSame(fieldC, colB.schema().getChildren().iterator().next()); Iterator<MaterializedField> iterC = colC.schema().getChildren().iterator(); - assertSame(fieldD, iterC.next()); - assertSame(fieldE, iterC.next()); + assertTrue(fieldD.isEquivalent(iterC.next())); + assertTrue(fieldE.isEquivalent(iterC.next())); // Copying should be deep. TupleMetadata root2 = ((TupleSchema) root).copy(); assertEquals(2, root2.metadata(0).mapSchema().metadata(0).mapSchema().metadata(0).mapSchema().size()); assert(root.isEquivalent(root2)); + + // Generate a materialized field and compare. + + fieldA.addChild(fieldB); + fieldB.addChild(fieldC); + fieldC.addChild(fieldD); + fieldC.addChild(fieldE); + assertTrue(colA.schema().isEquivalent(fieldA)); + } + + @Test + public void testMapTupleFromField() { + + // Create a materialized field with the desired structure. + + MaterializedField fieldA = SchemaBuilder.columnSchema("a", MinorType.MAP, DataMode.REQUIRED); + + MaterializedField fieldB = SchemaBuilder.columnSchema("b.x", MinorType.MAP, DataMode.REQUIRED); + fieldA.addChild(fieldB); + + MaterializedField fieldC = SchemaBuilder.columnSchema("c.y", MinorType.MAP, DataMode.REQUIRED); + fieldB.addChild(fieldC); + + MaterializedField fieldD = SchemaBuilder.columnSchema("d", MinorType.VARCHAR, DataMode.REQUIRED); + fieldC.addChild(fieldD); + + MaterializedField fieldE = SchemaBuilder.columnSchema("e", MinorType.INT, DataMode.REQUIRED); + fieldC.addChild(fieldE); + + // Create a metadata schema from the field. + + TupleMetadata root = new TupleSchema(); + ColumnMetadata colA = root.add(fieldA); + + // Get the parts. + + TupleMetadata mapA = colA.mapSchema(); + ColumnMetadata colB = mapA.metadata("b.x"); + TupleMetadata mapB = colB.mapSchema(); + ColumnMetadata colC = mapB.metadata("c.y"); + TupleMetadata mapC = colC.mapSchema(); + ColumnMetadata colD = mapC.metadata("d"); + ColumnMetadata colE = mapC.metadata("e"); + + // Validate. Should be same as earlier test that started + // with the metadata. + + assertEquals(1, root.size()); + assertEquals(1, mapA.size()); + assertEquals(1, mapB.size()); + assertEquals(2, mapC.size()); + + assertSame(colA, root.metadata("a")); + assertSame(colB, mapA.metadata("b.x")); + assertSame(colC, mapB.metadata("c.y")); + assertSame(colD, mapC.metadata("d")); + assertSame(colE, mapC.metadata("e")); + + // The full name contains quoted names if the contain dots. + // This name is more for diagnostic than semantic purposes. + + assertEquals("a", root.fullName(0)); + assertEquals("a.`b.x`", mapA.fullName(0)); + assertEquals("a.`b.x`.`c.y`", mapB.fullName(0)); + assertEquals("a.`b.x`.`c.y`.d", mapC.fullName(0)); + assertEquals("a.`b.x`.`c.y`.e", mapC.fullName(1)); + + assertEquals(1, colA.schema().getChildren().size()); + assertEquals(1, colB.schema().getChildren().size()); + assertEquals(2, colC.schema().getChildren().size()); + + assertTrue(colA.schema().isEquivalent(fieldA)); } } http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java index f0cc172..5b055af 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java @@ -17,7 +17,6 @@ */ package org.apache.drill.exec.store.easy.text.compliant; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -142,7 +141,7 @@ public class TestCsv extends ClusterTest { .add("b", MinorType.VARCHAR) .add("C", MinorType.VARCHAR) .build(); - assertEquals(expectedSchema, actual.batchSchema()); + assertTrue(expectedSchema.isEquivalent(actual.batchSchema())); RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema) .addRow("10", "foo", "bar") http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/test/DrillTestWrapper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/DrillTestWrapper.java b/exec/java-exec/src/test/java/org/apache/drill/test/DrillTestWrapper.java index 78e32ee..b5450e6 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/DrillTestWrapper.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/DrillTestWrapper.java @@ -350,7 +350,7 @@ public class DrillTestWrapper { for (VectorAccessible loader : batches) { numBatch++; if (expectedSchema != null) { - if (! expectedSchema.equals(loader.getSchema())) { + if (! expectedSchema.isEquivalent(loader.getSchema())) { throw new SchemaChangeException(String.format("Batch schema does not match expected schema\n" + "Actual schema: %s. Expected schema : %s", loader.getSchema(), expectedSchema)); @@ -469,8 +469,8 @@ public class DrillTestWrapper { final String expectedSchemaPath = expectedSchema.get(i).getLeft().getRootSegmentPath(); final TypeProtos.MajorType expectedMajorType = expectedSchema.get(i).getValue(); - if (!actualSchemaPath.equals(expectedSchemaPath) - || !actualMajorType.equals(expectedMajorType)) { + if (! actualSchemaPath.equals(expectedSchemaPath) || + ! Types.isEquivalent(actualMajorType, expectedMajorType)) { throw new Exception(String.format("Schema path or type mismatch for column #%d:\n" + "Expected schema path: %s\nActual schema path: %s\nExpected type: %s\nActual type: %s", i, expectedSchemaPath, actualSchemaPath, Types.toString(expectedMajorType), http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java index 2cd5d02..3e50f75 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java @@ -18,15 +18,10 @@ package org.apache.drill.test; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; -import com.google.common.base.Function; -import com.google.common.base.Preconditions; -import io.netty.buffer.DrillBuf; import org.apache.calcite.schema.SchemaPlus; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.scanner.ClassPathScanner; @@ -45,19 +40,17 @@ import org.apache.drill.exec.ops.BufferManager; import org.apache.drill.exec.ops.BufferManagerImpl; import org.apache.drill.exec.ops.ContextInformation; import org.apache.drill.exec.ops.FragmentContext; -import org.apache.drill.exec.ops.MetricDef; import org.apache.drill.exec.ops.OpProfileDef; import org.apache.drill.exec.ops.OperatorContext; -import org.apache.drill.exec.ops.OperatorStatReceiver; import org.apache.drill.exec.ops.OperatorStats; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.ExecProtos; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; -import org.apache.drill.exec.record.TupleMetadata; -import org.apache.drill.exec.record.TupleSchema; import org.apache.drill.exec.record.VectorContainer; +import org.apache.drill.exec.record.metadata.MetadataUtils; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.record.selection.SelectionVector2; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.server.options.SystemOptionManager; @@ -73,8 +66,12 @@ import org.apache.drill.test.rowSet.RowSet.ExtendableRowSet; import org.apache.drill.test.rowSet.RowSetBuilder; import org.apache.hadoop.security.UserGroupInformation; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; import com.google.common.util.concurrent.ListenableFuture; +import io.netty.buffer.DrillBuf; + /** * Test fixture for operator and (especially) "sub-operator" tests. * These are tests that are done without the full Drillbit server. @@ -375,7 +372,7 @@ public class OperatorFixture extends BaseFixture implements AutoCloseable { } public RowSetBuilder rowSetBuilder(BatchSchema schema) { - return rowSetBuilder(TupleSchema.fromFields(schema)); + return rowSetBuilder(MetadataUtils.fromFields(schema)); } public RowSetBuilder rowSetBuilder(TupleMetadata schema) { http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/AbstractRowSet.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/AbstractRowSet.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/AbstractRowSet.java index d128e4f..2c8491b 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/AbstractRowSet.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/AbstractRowSet.java @@ -19,9 +19,9 @@ package org.apache.drill.test.rowSet; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.record.BatchSchema; -import org.apache.drill.exec.record.TupleMetadata; import org.apache.drill.exec.record.VectorAccessible; import org.apache.drill.exec.record.VectorContainer; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.SchemaChangeCallBack; /** http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/AbstractSingleRowSet.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/AbstractSingleRowSet.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/AbstractSingleRowSet.java index ef41b3a..71ca3cf 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/AbstractSingleRowSet.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/AbstractSingleRowSet.java @@ -21,8 +21,8 @@ import org.apache.drill.exec.record.RecordBatchSizer; import org.apache.drill.exec.physical.rowSet.model.ReaderIndex; import org.apache.drill.exec.physical.rowSet.model.MetadataProvider.MetadataRetrieval; import org.apache.drill.exec.physical.rowSet.model.single.BaseReaderBuilder; -import org.apache.drill.exec.record.TupleMetadata; import org.apache.drill.exec.record.VectorContainer; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; /** http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/DirectRowSet.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/DirectRowSet.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/DirectRowSet.java index b5b1f1f..9262706 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/DirectRowSet.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/DirectRowSet.java @@ -19,18 +19,18 @@ package org.apache.drill.test.rowSet; import com.google.common.collect.Sets; import org.apache.drill.exec.memory.BufferAllocator; +import org.apache.drill.exec.physical.rowSet.model.MetadataProvider.MetadataRetrieval; import org.apache.drill.exec.physical.rowSet.model.ReaderIndex; import org.apache.drill.exec.physical.rowSet.model.SchemaInference; -import org.apache.drill.exec.physical.rowSet.model.MetadataProvider.MetadataRetrieval; import org.apache.drill.exec.physical.rowSet.model.single.BaseWriterBuilder; import org.apache.drill.exec.physical.rowSet.model.single.BuildVectorsFromMetadata; import org.apache.drill.exec.physical.rowSet.model.single.VectorAllocator; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; -import org.apache.drill.exec.record.TupleMetadata; -import org.apache.drill.exec.record.TupleSchema; import org.apache.drill.exec.record.VectorAccessible; import org.apache.drill.exec.record.VectorContainer; +import org.apache.drill.exec.record.metadata.MetadataUtils; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.record.selection.SelectionVector2; import org.apache.drill.test.rowSet.RowSet.ExtendableRowSet; import org.apache.drill.test.rowSet.RowSetWriterImpl.WriterIndexImpl; @@ -85,7 +85,7 @@ public class DirectRowSet extends AbstractSingleRowSet implements ExtendableRowS } public static DirectRowSet fromSchema(BufferAllocator allocator, BatchSchema schema) { - return fromSchema(allocator, TupleSchema.fromFields(schema)); + return fromSchema(allocator, MetadataUtils.fromFields(schema)); } public static DirectRowSet fromSchema(BufferAllocator allocator, TupleMetadata schema) { http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/HyperRowSetImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/HyperRowSetImpl.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/HyperRowSetImpl.java index d0ca662..70b8a20 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/HyperRowSetImpl.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/HyperRowSetImpl.java @@ -21,7 +21,7 @@ import org.apache.drill.exec.physical.rowSet.model.MetadataProvider.MetadataRetr import org.apache.drill.exec.physical.rowSet.model.SchemaInference; import org.apache.drill.exec.physical.rowSet.model.hyper.BaseReaderBuilder; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; -import org.apache.drill.exec.record.TupleMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.record.VectorContainer; import org.apache.drill.exec.record.selection.SelectionVector4; import org.apache.drill.test.rowSet.RowSet.HyperRowSet; http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java index ec0925e..53be75d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java @@ -20,7 +20,7 @@ package org.apache.drill.test.rowSet; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; -import org.apache.drill.exec.record.TupleMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.record.VectorAccessible; import org.apache.drill.exec.record.VectorContainer; import org.apache.drill.exec.record.selection.SelectionVector2; http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java index d6bbaf8..c1a98ac 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java @@ -20,8 +20,8 @@ package org.apache.drill.test.rowSet; import com.google.common.collect.Sets; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.record.BatchSchema; -import org.apache.drill.exec.record.TupleMetadata; -import org.apache.drill.exec.record.TupleSchema; +import org.apache.drill.exec.record.metadata.MetadataUtils; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.accessor.TupleWriter; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; @@ -46,7 +46,7 @@ public final class RowSetBuilder { private Set<Integer> skipIndices = Sets.newHashSet(); public RowSetBuilder(BufferAllocator allocator, BatchSchema schema) { - this(allocator, TupleSchema.fromFields(schema), 10); + this(allocator, MetadataUtils.fromFields(schema), 10); } public RowSetBuilder(BufferAllocator allocator, TupleMetadata schema) { http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java index e730987..fe50197 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java @@ -20,8 +20,8 @@ package org.apache.drill.test.rowSet; import java.io.PrintStream; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; -import org.apache.drill.exec.record.ColumnMetadata; -import org.apache.drill.exec.record.TupleMetadata; +import org.apache.drill.exec.record.metadata.ColumnMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; /** * Print a row set in CSV-like format. Primarily for debugging. http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReaderImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReaderImpl.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReaderImpl.java index 2bae085..7217187 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReaderImpl.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReaderImpl.java @@ -20,7 +20,7 @@ package org.apache.drill.test.rowSet; import java.util.List; import org.apache.drill.exec.physical.rowSet.model.ReaderIndex; -import org.apache.drill.exec.record.TupleMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.accessor.reader.AbstractObjectReader; import org.apache.drill.exec.vector.accessor.reader.AbstractTupleReader; http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriterImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriterImpl.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriterImpl.java index 074842d..b649b11 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriterImpl.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriterImpl.java @@ -19,7 +19,7 @@ package org.apache.drill.test.rowSet; import java.util.List; -import org.apache.drill.exec.record.TupleMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; import org.apache.drill.exec.vector.accessor.writer.AbstractObjectWriter; http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/SchemaBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/SchemaBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/SchemaBuilder.java index 9223ef4..3d4df05 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/SchemaBuilder.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/SchemaBuilder.java @@ -22,11 +22,12 @@ import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; -import org.apache.drill.exec.record.ColumnMetadata; import org.apache.drill.exec.record.MaterializedField; -import org.apache.drill.exec.record.TupleMetadata; -import org.apache.drill.exec.record.TupleSchema; -import org.apache.drill.exec.record.TupleSchema.MapColumnMetadata; +import org.apache.drill.exec.record.metadata.ColumnMetadata; +import org.apache.drill.exec.record.metadata.MapColumnMetadata; +import org.apache.drill.exec.record.metadata.MetadataUtils; +import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.exec.record.metadata.TupleSchema; /** * Builder of a row set schema expressed as a list of materialized @@ -128,7 +129,7 @@ public class SchemaBuilder { for (ColumnMetadata md : schema) { col.addChild(md.schema()); } - parent.finishMap(TupleSchema.newMap(col, schema)); + parent.finishMap(MetadataUtils.newMap(col, schema)); return parent; } http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/DummyWriterTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/DummyWriterTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/DummyWriterTest.java index db33b30..46f1cc3 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/DummyWriterTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/DummyWriterTest.java @@ -23,7 +23,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.drill.common.types.TypeProtos.MinorType; -import org.apache.drill.exec.record.TupleMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.accessor.writer.AbstractObjectWriter; import org.apache.drill.exec.vector.accessor.writer.AbstractTupleWriter; import org.apache.drill.exec.vector.accessor.writer.ColumnWriterFactory; http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/PerformanceTool.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/PerformanceTool.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/PerformanceTool.java index ca282a1..4819253 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/PerformanceTool.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/PerformanceTool.java @@ -21,10 +21,10 @@ import java.util.concurrent.TimeUnit; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; -import org.apache.drill.exec.record.ColumnMetadata; import org.apache.drill.exec.record.MaterializedField; -import org.apache.drill.exec.record.TupleMetadata; -import org.apache.drill.exec.record.TupleSchema; +import org.apache.drill.exec.record.metadata.ColumnMetadata; +import org.apache.drill.exec.record.metadata.MetadataUtils; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.IntVector; import org.apache.drill.exec.vector.NullableIntVector; import org.apache.drill.exec.vector.RepeatedIntVector; @@ -248,7 +248,7 @@ public class PerformanceTool { try (RepeatedIntVector vector = new RepeatedIntVector(rowSchema.column(0), fixture.allocator());) { vector.allocateNew(ROW_COUNT, 5 * ROW_COUNT); IntColumnWriter colWriter = new IntColumnWriter(vector.getDataVector()); - ColumnMetadata colSchema = TupleSchema.fromField(vector.getField()); + ColumnMetadata colSchema = MetadataUtils.fromField(vector.getField()); ArrayObjectWriter arrayWriter = ScalarArrayWriter.build(colSchema, vector, colWriter); TestWriterIndex index = new TestWriterIndex(); arrayWriter.events().bindIndex(index); http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java index 4db4d09..5ba1c54 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java @@ -28,7 +28,7 @@ import java.util.Arrays; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.BatchSchema; -import org.apache.drill.exec.record.TupleMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.ArrayReader; import org.apache.drill.exec.vector.accessor.ArrayWriter; http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java index 147b713..f9f1b8a 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java @@ -23,7 +23,7 @@ import static org.junit.Assert.assertTrue; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; -import org.apache.drill.exec.record.TupleMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.accessor.ScalarElementReader; import org.apache.drill.exec.vector.accessor.ScalarReader; import org.apache.drill.exec.vector.accessor.ScalarWriter; http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/vector/src/main/java/org/apache/drill/exec/record/ColumnMetadata.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/ColumnMetadata.java b/exec/vector/src/main/java/org/apache/drill/exec/record/ColumnMetadata.java deleted file mode 100644 index 558aab8..0000000 --- a/exec/vector/src/main/java/org/apache/drill/exec/record/ColumnMetadata.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.record; - -import org.apache.drill.common.types.TypeProtos.DataMode; -import org.apache.drill.common.types.TypeProtos.MajorType; -import org.apache.drill.common.types.TypeProtos.MinorType; - -/** - * Metadata description of a column including names, types and structure - * information. - */ - -public interface ColumnMetadata { - enum StructureType { - PRIMITIVE, LIST, TUPLE - } - - public static final int DEFAULT_ARRAY_SIZE = 10; - - ColumnMetadata.StructureType structureType(); - TupleMetadata mapSchema(); - MaterializedField schema(); - String name(); - MajorType majorType(); - MinorType type(); - DataMode mode(); - boolean isNullable(); - boolean isArray(); - boolean isVariableWidth(); - boolean isMap(); - boolean isList(); - - /** - * Report whether one column is equivalent to another. Columns are equivalent - * if they have the same name, type and structure (ignoring internal structure - * such as offset vectors.) - */ - - boolean isEquivalent(ColumnMetadata other); - - /** - * For variable-width columns, specify the expected column width to be used - * when allocating a new vector. Does nothing for fixed-width columns. - * - * @param width the expected column width - */ - - void setExpectedWidth(int width); - - /** - * Get the expected width for a column. This is the actual width for fixed- - * width columns, the specified width (defaulting to 50) for variable-width - * columns. - * @return the expected column width of the each data value. Does not include - * "overhead" space such as for the null-value vector or offset vector - */ - - int expectedWidth(); - - /** - * For an array column, specify the expected average array cardinality. - * Ignored for non-array columns. Used when allocating new vectors. - * - * @param childCount the expected average array cardinality. Defaults to - * 1 for non-array columns, 10 for array columns - */ - - void setExpectedElementCount(int childCount); - - /** - * Returns the expected array cardinality for array columns, or 1 for - * non-array columns. - * - * @return the expected value cardinality per value (per-row for top-level - * columns, per array element for arrays within lists) - */ - - int expectedElementCount(); - - /** - * Create an empty version of this column. If the column is a scalar, - * produces a simple copy. If a map, produces a clone without child - * columns. - * - * @return empty clone of this column - */ - - ColumnMetadata cloneEmpty(); - - /** - * Reports whether, in this context, the column is projected outside - * of the context. (That is, whether the column is backed by an actual - * value vector.) - */ - - boolean isProjected(); - void setProjected(boolean projected); -} http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/vector/src/main/java/org/apache/drill/exec/record/TupleMetadata.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/TupleMetadata.java b/exec/vector/src/main/java/org/apache/drill/exec/record/TupleMetadata.java deleted file mode 100644 index 8f597be..0000000 --- a/exec/vector/src/main/java/org/apache/drill/exec/record/TupleMetadata.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.record; - -import java.util.List; - -/** - * Metadata description of the schema of a row or a map. - * In Drill, both rows and maps are - * tuples: both are an ordered collection of values, defined by a - * schema. Each tuple has a schema that defines the column ordering - * for indexed access. Each tuple also provides methods to get column - * accessors by name or index. - * <p> - * Models the physical schema of a row set showing the logical hierarchy of fields - * with map fields as first-class fields. Map members appear as children - * under the map, much as they appear in the physical value-vector - * implementation. - * <ul> - * <li>Provides fast lookup by name or index.</li> - * <li>Provides a nested schema, in this same form, for maps.</li> - * </ul> - * This form is useful when performing semantic analysis and when - * working with vectors. - * <p> - * In the future, this structure will also gather metadata useful - * for vector processing such as expected widths and so on. - */ - -public interface TupleMetadata extends Iterable<ColumnMetadata> { - - /** - * Add a new column to the schema. - * - * @param columnSchema - * @return the index of the new column - */ - ColumnMetadata add(MaterializedField field); - int addColumn(ColumnMetadata column); - - int size(); - boolean isEmpty(); - int index(String name); - ColumnMetadata metadata(int index); - ColumnMetadata metadata(String name); - MaterializedField column(int index); - MaterializedField column(String name); - boolean isEquivalent(TupleMetadata other); - ColumnMetadata parent(); - - /** - * Return the schema as a list of <tt>MaterializedField</tt> objects - * which can be used to create other schemas. Not valid for a - * flattened schema. - * - * @return a list of the top-level fields. Maps contain their child - * fields - */ - - List<MaterializedField> toFieldList(); - - /** - * Full name of the column. Note: this name cannot be used to look up - * the column because of ambiguity. The name "a.b.c" may mean a single - * column with that name, or may mean maps "a", and "b" with column "c", - * etc. - * - * @return full, dotted, column name - */ - - String fullName(ColumnMetadata column); - String fullName(int index); -} http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/vector/src/main/java/org/apache/drill/exec/record/TupleNameSpace.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/TupleNameSpace.java b/exec/vector/src/main/java/org/apache/drill/exec/record/TupleNameSpace.java deleted file mode 100644 index 5853c93..0000000 --- a/exec/vector/src/main/java/org/apache/drill/exec/record/TupleNameSpace.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.record; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import org.apache.drill.common.map.CaseInsensitiveMap; - -import com.google.common.collect.ImmutableList; - -/** - * Implementation of a tuple name space. Tuples allow both indexed and - * named access to their members. - * - * @param <T> the type of object representing each column - */ - -public class TupleNameSpace<T> implements Iterable<T> { - private final Map<String,Integer> nameSpace = CaseInsensitiveMap.newHashMap(); - private final List<T> entries = new ArrayList<>(); - - public int add(String key, T value) { - if (indexOf(key) != -1) { - throw new IllegalArgumentException("Duplicate entry: " + key); - } - int index = entries.size(); - nameSpace.put(key, index); - entries.add(value); - return index; - } - - public T get(int index) { - return entries.get(index); - } - - public T get(String key) { - int index = indexOf(key); - if (index == -1) { - return null; - } - return get(index); - } - - public int indexOf(String key) { - Integer index = nameSpace.get(key); - if (index == null) { - return -1; - } - return index; - } - - public int count() { return entries.size(); } - - @Override - public Iterator<T> iterator() { - return entries.iterator(); - } - - public boolean isEmpty() { - return entries.isEmpty(); - } - - public List<T> entries() { - return ImmutableList.copyOf(entries); - } - - @Override - public String toString() { - return entries.toString(); - } -} http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ColumnMetadata.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ColumnMetadata.java b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ColumnMetadata.java new file mode 100644 index 0000000..70a60e6 --- /dev/null +++ b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ColumnMetadata.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.record.metadata; + +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.MaterializedField; + +/** + * Metadata description of a column including names, types and structure + * information. + */ + +public interface ColumnMetadata { + + /** + * Rough characterization of Drill types into metadata categories. + * Various aspects of Drill's type system are very, very messy. + * However, Drill is defined by its code, not some abstract design, + * so the metadata system here does the best job it can to simplify + * the messy type system while staying close to the underlying + * implementation. + */ + + enum StructureType { + + /** + * Primitive column (all types except List, Map and Union.) + * Includes (one-dimensional) arrays of those types. + */ + + PRIMITIVE, + + /** + * Map or repeated map. Also describes the row as a whole. + */ + + TUPLE, + + /** + * Union or (non-repeated) list. (A non-repeated list is, + * essentially, a repeated union.) + */ + + VARIANT, + + /** + * A repeated list. A repeated list is not simply the repeated + * form of a list, it is something else entirely. It acts as + * a dimensional wrapper around any other type (except list) + * and adds a non-nullable extra dimension. Hence, this type is + * for 2D+ arrays. + * <p> + * In theory, a 2D list of, say, INT would be an INT column, but + * repeated in to dimensions. Alas, that is not how it is. Also, + * if we have a separate category for 2D lists, we should have + * a separate category for 1D lists. But, again, that is not how + * the code has evolved. + */ + + MULTI_ARRAY + } + + int DEFAULT_ARRAY_SIZE = 10; + + StructureType structureType(); + + /** + * Schema for <tt>TUPLE</tt> columns. + * + * @return the tuple schema + */ + + TupleMetadata mapSchema(); + + /** + * Schema for <tt>VARIANT</tt> columns. + * + * @return the variant schema + */ + + VariantMetadata variantSchema(); + + /** + * Schema of inner dimension for <tt>MULTI_ARRAY<tt> columns. + * If an array is 3D, the outer column represents all 3 dimensions. + * <tt>outer.childSchema()</tt> gives another <tt>MULTI_ARRAY</tt> + * for the inner 2D array. + * <tt>outer.childSchema().childSchema()</tt> gives a column + * of some other type (but repeated) for the 1D array. + * <p> + * Sorry for the mess, but it is how the code works and we are not + * in a position to revisit data type fundamentals. + * + * @return the description of the (n-1) st dimension. + */ + + ColumnMetadata childSchema(); + MaterializedField schema(); + MaterializedField emptySchema(); + String name(); + MinorType type(); + MajorType majorType(); + DataMode mode(); + int dimensions(); + boolean isNullable(); + boolean isArray(); + boolean isVariableWidth(); + boolean isMap(); + boolean isVariant(); + + /** + * Report whether one column is equivalent to another. Columns are equivalent + * if they have the same name, type and structure (ignoring internal structure + * such as offset vectors.) + */ + + boolean isEquivalent(ColumnMetadata other); + + /** + * For variable-width columns, specify the expected column width to be used + * when allocating a new vector. Does nothing for fixed-width columns. + * + * @param width the expected column width + */ + + void setExpectedWidth(int width); + + /** + * Get the expected width for a column. This is the actual width for fixed- + * width columns, the specified width (defaulting to 50) for variable-width + * columns. + * @return the expected column width of the each data value. Does not include + * "overhead" space such as for the null-value vector or offset vector + */ + + int expectedWidth(); + + /** + * For an array column, specify the expected average array cardinality. + * Ignored for non-array columns. Used when allocating new vectors. + * + * @param childCount the expected average array cardinality. Defaults to + * 1 for non-array columns, 10 for array columns + */ + + void setExpectedElementCount(int childCount); + + /** + * Returns the expected array cardinality for array columns, or 1 for + * non-array columns. + * + * @return the expected value cardinality per value (per-row for top-level + * columns, per array element for arrays within lists) + */ + + int expectedElementCount(); + + /** + * Create an empty version of this column. If the column is a scalar, + * produces a simple copy. If a map, produces a clone without child + * columns. + * + * @return empty clone of this column + */ + + ColumnMetadata cloneEmpty(); + + /** + * Reports whether, in this context, the column is projected outside + * of the context. (That is, whether the column is backed by an actual + * value vector.) + */ + + boolean isProjected(); + void setProjected(boolean projected); + + int precision(); + int scale(); +} http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ProjectionType.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ProjectionType.java b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ProjectionType.java new file mode 100644 index 0000000..4972605 --- /dev/null +++ b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ProjectionType.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.record.metadata; + +public enum ProjectionType { + UNPROJECTED, + WILDCARD, // * + UNSPECIFIED, // x + TUPLE, // x.y + ARRAY, // x[0] + TUPLE_ARRAY // x[0].y +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/TupleMetadata.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/TupleMetadata.java b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/TupleMetadata.java new file mode 100644 index 0000000..a65c9f2 --- /dev/null +++ b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/TupleMetadata.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.record.metadata; + +import java.util.List; + +import org.apache.drill.exec.record.MaterializedField; + +/** + * Metadata description of the schema of a row or a map. + * In Drill, both rows and maps are + * tuples: both are an ordered collection of values, defined by a + * schema. Each tuple has a schema that defines the column ordering + * for indexed access. Each tuple also provides methods to get column + * accessors by name or index. + * <p> + * Models the physical schema of a row set showing the logical hierarchy of fields + * with map fields as first-class fields. Map members appear as children + * under the map, much as they appear in the physical value-vector + * implementation. + * <ul> + * <li>Provides fast lookup by name or index.</li> + * <li>Provides a nested schema, in this same form, for maps.</li> + * </ul> + * This form is useful when performing semantic analysis and when + * working with vectors. + * <p> + * In the future, this structure will also gather metadata useful + * for vector processing such as expected widths and so on. + */ + +public interface TupleMetadata extends Iterable<ColumnMetadata> { + + /** + * Add a new column to the schema. + * + * @param columnSchema + * @return the index of the new column + */ + ColumnMetadata add(MaterializedField field); + int addColumn(ColumnMetadata column); + + int size(); + boolean isEmpty(); + int index(String name); + ColumnMetadata metadata(int index); + ColumnMetadata metadata(String name); + MaterializedField column(int index); + MaterializedField column(String name); + boolean isEquivalent(TupleMetadata other); + ColumnMetadata parent(); + + /** + * Return the schema as a list of <tt>MaterializedField</tt> objects + * which can be used to create other schemas. Not valid for a + * flattened schema. + * + * @return a list of the top-level fields. Maps contain their child + * fields + */ + + List<MaterializedField> toFieldList(); + + /** + * Full name of the column. Note: this name cannot be used to look up + * the column because of ambiguity. The name "a.b.c" may mean a single + * column with that name, or may mean maps "a", and "b" with column "c", + * etc. + * + * @return full, dotted, column name + */ + + String fullName(ColumnMetadata column); + String fullName(int index); +} http://git-wip-us.apache.org/repos/asf/drill/blob/cf2478f7/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/TupleNameSpace.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/TupleNameSpace.java b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/TupleNameSpace.java new file mode 100644 index 0000000..3379639 --- /dev/null +++ b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/TupleNameSpace.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.record.metadata; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.drill.common.map.CaseInsensitiveMap; + +import com.google.common.collect.ImmutableList; + +/** + * Implementation of a tuple name space. Tuples allow both indexed and + * named access to their members. + * + * @param <T> the type of object representing each column + */ + +public class TupleNameSpace<T> implements Iterable<T> { + private final Map<String,Integer> nameSpace = CaseInsensitiveMap.newHashMap(); + private final List<T> entries = new ArrayList<>(); + + public int add(String key, T value) { + if (indexOf(key) != -1) { + throw new IllegalArgumentException("Duplicate entry: " + key); + } + int index = entries.size(); + nameSpace.put(key, index); + entries.add(value); + return index; + } + + public T get(int index) { + return entries.get(index); + } + + public T get(String key) { + int index = indexOf(key); + if (index == -1) { + return null; + } + return get(index); + } + + public int indexOf(String key) { + Integer index = nameSpace.get(key); + if (index == null) { + return -1; + } + return index; + } + + public int count() { return entries.size(); } + + @Override + public Iterator<T> iterator() { + return entries.iterator(); + } + + public boolean isEmpty() { + return entries.isEmpty(); + } + + public List<T> entries() { + return ImmutableList.copyOf(entries); + } + + @Override + public String toString() { + return entries.toString(); + } +}
