This is an automated email from the ASF dual-hosted git repository. volodymyr pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
commit 87a6725718446b316d85d5dab1d3ad135f5d9475 Author: Arina Ielchiieva <[email protected]> AuthorDate: Tue Mar 10 16:06:57 2020 +0200 DRILL-7630: Add additional types into SchemaParser for Parquet closes #2019 --- .../record/metadata/schema/parser/SchemaLexer.g4 | 8 ++++++ .../record/metadata/schema/parser/SchemaParser.g4 | 6 +++++ .../metadata/schema/parser/SchemaVisitor.java | 30 ++++++++++++++++++++++ .../metadata/schema/parser/TestSchemaParser.java | 15 +++++++++++ 4 files changed, 59 insertions(+) diff --git a/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaLexer.g4 b/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaLexer.g4 index 62d9f10..c531356 100644 --- a/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaLexer.g4 +++ b/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaLexer.g4 @@ -58,6 +58,14 @@ MAP: 'MAP'; STRUCT: 'STRUCT'; ARRAY: 'ARRAY'; +// additional data types, primary used for Parquet +UINT1: 'UINT1'; +UINT2: 'UINT2'; +UINT4: 'UINT4'; +UINT8: 'UINT8'; +TINYINT: 'TINYINT'; +SMALLINT: 'SMALLINT'; + // symbols COMMA: ','; REVERSE_QUOTE: '`'; diff --git a/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaParser.g4 b/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaParser.g4 index 5bd55a7..1ff90ec 100644 --- a/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaParser.g4 +++ b/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaParser.g4 @@ -63,6 +63,12 @@ simple_type | INTERVAL (YEAR | MONTH) # interval_year | INTERVAL (DAY | HOUR | MINUTE | SECOND) # interval_day | INTERVAL # interval +| UINT1 # unit1 +| UINT2 # unit2 +| UINT4 # unit4 +| UINT8 # unit8 +| TINYINT # tinyint +| SMALLINT # smallint ; array_type: (simple_array_type | complex_array_type); diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/schema/parser/SchemaVisitor.java b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/schema/parser/SchemaVisitor.java index cd5da79..18ed651 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/schema/parser/SchemaVisitor.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/schema/parser/SchemaVisitor.java @@ -279,6 +279,36 @@ public class SchemaVisitor extends SchemaParserBaseVisitor<TupleMetadata> { } @Override + public ColumnMetadata visitUnit1(SchemaParser.Unit1Context ctx) { + return constructColumn(Types.withMode(TypeProtos.MinorType.UINT1, mode)); + } + + @Override + public ColumnMetadata visitUnit2(SchemaParser.Unit2Context ctx) { + return constructColumn(Types.withMode(TypeProtos.MinorType.UINT2, mode)); + } + + @Override + public ColumnMetadata visitUnit4(SchemaParser.Unit4Context ctx) { + return constructColumn(Types.withMode(TypeProtos.MinorType.UINT4, mode)); + } + + @Override + public ColumnMetadata visitUnit8(SchemaParser.Unit8Context ctx) { + return constructColumn(Types.withMode(TypeProtos.MinorType.UINT8, mode)); + } + + @Override + public ColumnMetadata visitTinyint(SchemaParser.TinyintContext ctx) { + return constructColumn(Types.withMode(TypeProtos.MinorType.TINYINT, mode)); + } + + @Override + public ColumnMetadata visitSmallint(SchemaParser.SmallintContext ctx) { + return constructColumn(Types.withMode(TypeProtos.MinorType.SMALLINT, mode)); + } + + @Override public ColumnMetadata visitStruct_type(SchemaParser.Struct_typeContext ctx) { // internally Drill refers to structs as maps MapBuilder builder = new MapBuilder(null, name, mode); diff --git a/exec/vector/src/test/java/org/apache/drill/exec/record/metadata/schema/parser/TestSchemaParser.java b/exec/vector/src/test/java/org/apache/drill/exec/record/metadata/schema/parser/TestSchemaParser.java index a0acf10..201a2dc 100644 --- a/exec/vector/src/test/java/org/apache/drill/exec/record/metadata/schema/parser/TestSchemaParser.java +++ b/exec/vector/src/test/java/org/apache/drill/exec/record/metadata/schema/parser/TestSchemaParser.java @@ -185,6 +185,21 @@ public class TestSchemaParser extends BaseTest { } @Test + public void testAdditionalParquetTypes() throws Exception { + TupleMetadata schema = new SchemaBuilder() + .addNullable("uint1_col", TypeProtos.MinorType.UINT1) + .addNullable("uint2_col", TypeProtos.MinorType.UINT2) + .addNullable("uint4_col", TypeProtos.MinorType.UINT4) + .addNullable("uint8_col", TypeProtos.MinorType.UINT8) + .addNullable("tinyint_col", TypeProtos.MinorType.TINYINT) + .addNullable("smallint_col", TypeProtos.MinorType.SMALLINT) + .buildSchema(); + + checkSchema("uint1_col uint1, uint2_col uint2, uint4_col uint4, uint8_col uint8, " + + "tinyint_col tinyint, smallint_col smallint", schema); + } + + @Test public void testArray() throws Exception { TupleMetadata schema = new SchemaBuilder() .addArray("simple_array", TypeProtos.MinorType.INT)
