Repository: tajo Updated Branches: refs/heads/master 4253f1b60 -> a94936ae5
TAJO-1737: Implement SQL Parser rule for Map type. Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/08efcf27 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/08efcf27 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/08efcf27 Branch: refs/heads/master Commit: 08efcf279140b4f5bd0e5aca4de19e792fc961a9 Parents: d8ce562 Author: Hyunsik Choi <[email protected]> Authored: Mon Aug 3 20:33:17 2015 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Mon Aug 3 20:33:17 2015 +0900 ---------------------------------------------------------------------- .../apache/tajo/algebra/ColumnDefinition.java | 9 ++- .../org/apache/tajo/algebra/DataTypeExpr.java | 81 +++++++++++++++++--- tajo-common/src/main/proto/DataTypes.proto | 1 + .../org/apache/tajo/engine/parser/SQLLexer.g4 | 1 + .../org/apache/tajo/engine/parser/SQLParser.g4 | 6 ++ .../apache/tajo/engine/parser/SQLAnalyzer.java | 10 ++- .../tajo/engine/parser/TestSQLAnalyzer.java | 18 +++++ .../TestSQLAnalyzer/create_table_maptype_1.sql | 1 + .../TestSQLAnalyzer/create_table_maptype_2.sql | 1 + .../TestSQLAnalyzer/create_table_maptype_3.sql | 1 + .../create_table_maptype_1.result | 38 +++++++++ .../create_table_maptype_2.result | 52 +++++++++++++ .../create_table_maptype_3.result | 66 ++++++++++++++++ 13 files changed, 272 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java index f8ea0f1..2a829e1 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java @@ -43,8 +43,13 @@ public class ColumnDefinition extends DataTypeExpr { } // nested records - if (dataType.isNestedRecordType()) { - this.nestedRecordTypes = dataType.nestedRecordTypes; + if (dataType.isRecordType()) { + this.recordType = dataType.recordType; + } + + // map type + if (dataType.isMapType()) { + this.mapType = dataType.mapType; } } http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java index b280397..d63532d 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java @@ -32,30 +32,46 @@ public class DataTypeExpr extends Expr { @Expose @SerializedName("Scale") Integer scale; @Expose @SerializedName("Record") - ColumnDefinition [] nestedRecordTypes; // not null if the type is RECORD + RecordType recordType; // not null if the type is RECORD + @Expose @SerializedName("Map") + MapType mapType; public DataTypeExpr(String typeName) { super(OpType.DataType); this.typeName = typeName; } - public DataTypeExpr(ColumnDefinition [] nestedRecordTypes) { + public DataTypeExpr(RecordType record) { super(OpType.DataType); - // RECORD = 51 in DataTypes.proto this.typeName = Type.RECORD.name(); - this.nestedRecordTypes = nestedRecordTypes; + this.recordType = record; + } + + public DataTypeExpr(MapType map) { + super(OpType.DataType); + // RECORD = 51 in DataTypes.proto + this.typeName = Type.MAP.name(); + this.mapType = map; } public String getTypeName() { return this.typeName; } - public boolean isNestedRecordType() { + public boolean isPrimitiveType() { + return !this.isRecordType() && !isMapType(); + } + + public boolean isRecordType() { return this.typeName.equals(Type.RECORD.name()); } + public boolean isMapType() { + return this.typeName.equals(Type.MAP.name()); + } + public ColumnDefinition [] getNestedRecordTypes() { - return nestedRecordTypes; + return recordType.schema; } public boolean hasLengthOrPrecision() { @@ -84,7 +100,7 @@ public class DataTypeExpr extends Expr { @Override public int hashCode() { - return Objects.hashCode(typeName, lengthOrPrecision, scale); + return Objects.hashCode(typeName, lengthOrPrecision, scale, recordType, mapType); } @Override @@ -93,16 +109,63 @@ public class DataTypeExpr extends Expr { return typeName.equals(another.typeName) && TUtil.checkEquals(lengthOrPrecision, another.lengthOrPrecision) && TUtil.checkEquals(scale, another.scale) && - TUtil.checkEquals(nestedRecordTypes, another.nestedRecordTypes); + TUtil.checkEquals(recordType, another.recordType) && + TUtil.checkEquals(mapType, another.mapType); } @Override public Object clone() throws CloneNotSupportedException { DataTypeExpr dataType = (DataTypeExpr) super.clone(); dataType.typeName = typeName; + // why we copy references? because they are all immutable. dataType.lengthOrPrecision = lengthOrPrecision; dataType.scale = scale; - dataType.nestedRecordTypes = nestedRecordTypes; + dataType.recordType = recordType; + dataType.mapType = mapType; return dataType; } + + public static class RecordType implements JsonSerializable, Cloneable { + @Expose @SerializedName("Schema") + ColumnDefinition [] schema; // not null if the type is RECORD + + public RecordType(ColumnDefinition [] schema) { + this.schema = schema; + } + + @Override + public String toJson() { + return JsonHelper.toJson(this); + } + + public Object clone() throws CloneNotSupportedException { + RecordType newRecord = (RecordType) super.clone(); + newRecord.schema = this.schema; + return newRecord; + } + } + + public static class MapType implements JsonSerializable, Cloneable { + @Expose @SerializedName("KeyType") + DataTypeExpr keyType; + @Expose @SerializedName("ValueType") + DataTypeExpr valueType; + + public MapType(DataTypeExpr key, DataTypeExpr value) { + this.keyType = key; + this.valueType = value; + } + + @Override + public String toJson() { + return JsonHelper.toJson(this); + } + + public Object clone() throws CloneNotSupportedException { + MapType newMap = (MapType) super.clone(); + newMap.keyType = keyType; + newMap.valueType = valueType; + return newMap; + } + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-common/src/main/proto/DataTypes.proto ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/proto/DataTypes.proto b/tajo-common/src/main/proto/DataTypes.proto index fc5ac9a..53c3d0f 100644 --- a/tajo-common/src/main/proto/DataTypes.proto +++ b/tajo-common/src/main/proto/DataTypes.proto @@ -60,6 +60,7 @@ enum Type { BLOB = 45; RECORD = 51; // nested structure type + MAP = 52; // map type ANY = 61; // Any type UDT = 62; // user-defined function http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 index d1daeb6..896f627 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 @@ -258,6 +258,7 @@ LESS : L E S S; LIST : L I S T; LOCATION : L O C A T I O N; +MAP : M A P; MAX : M A X; MAXVALUE : M A X V A L U E; MICROSECONDS : M I C R O S E C O N D S; http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index 098994d..149269e 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -282,6 +282,7 @@ nonreserved_keywords | LESS | LIST | LOCATION + | MAP | MAX | MAXVALUE | MICROSECONDS @@ -441,6 +442,7 @@ predefined_type | binary_type | network_type | record_type + | map_type ; character_string_type @@ -540,6 +542,10 @@ record_type : RECORD table_elements ; +map_type + : MAP LTH key_type=data_type COMMA value_type=data_type GTH + ; + /* =============================================================================== 6.3 <value_expression_primary> http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index ffe5d2e..465aaa8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -28,6 +28,7 @@ import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.*; import org.apache.tajo.algebra.Aggregation.GroupType; import org.apache.tajo.algebra.CreateIndex.IndexMethodSpec; +import org.apache.tajo.algebra.DataTypeExpr.MapType; import org.apache.tajo.algebra.LiteralValue.LiteralType; import org.apache.tajo.algebra.Sort.SortSpec; import org.apache.tajo.engine.parser.SQLParser.*; @@ -1603,8 +1604,13 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> { } else if (checkIfExist(predefined_type.record_type())) { - ColumnDefinition [] nestedRecordDefines = getDefinitions(predefined_type.record_type().table_elements()); - typeDefinition = new DataTypeExpr(nestedRecordDefines); + ColumnDefinition [] nestedRecordDefine = getDefinitions(predefined_type.record_type().table_elements()); + typeDefinition = new DataTypeExpr(new DataTypeExpr.RecordType(nestedRecordDefine)); + + } else if (checkIfExist(predefined_type.map_type())) { + Map_typeContext mapTypeContext = predefined_type.map_type(); + typeDefinition = new DataTypeExpr( + new MapType(visitData_type(mapTypeContext.key_type), visitData_type(mapTypeContext.value_type))); } return typeDefinition; http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java index bb14aec..e9a9305 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java @@ -801,4 +801,22 @@ public class TestSQLAnalyzer { public void testCreateTableWithNested2() throws IOException { assertParseResult("create_table_nested_2.sql", "create_table_nested_2.result"); } + + @Test + public void testCreateTableWithMapType1() throws IOException { + // primitive key value map + assertParseResult("create_table_maptype_1.sql", "create_table_maptype_1.result"); + } + + @Test + public void testCreateTableWithMapType2() throws IOException { + // primitive key and record value map + assertParseResult("create_table_maptype_2.sql", "create_table_maptype_2.result"); + } + + @Test + public void testCreateTableWithMapType3() throws IOException { + // primitive key and nexted record value map + assertParseResult("create_table_maptype_3.sql", "create_table_maptype_3.result"); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_1.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_1.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_1.sql new file mode 100644 index 0000000..4a9789c --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_1.sql @@ -0,0 +1 @@ +CREATE TABLE T1 (A TEXT, B INT4, C MAP<TEXT, INT8>, F FLOAT8); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_2.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_2.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_2.sql new file mode 100644 index 0000000..9a828ce --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_2.sql @@ -0,0 +1 @@ +CREATE TABLE T1 (A TEXT, B INT4, C MAP<TEXT, RECORD (name TEXT, age INT4)>, F FLOAT8); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_3.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_3.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_3.sql new file mode 100644 index 0000000..fee6829 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_3.sql @@ -0,0 +1 @@ +CREATE TABLE T1 (A TEXT, B INT4, C MAP<TEXT, RECORD (name RECORD (first_name TEXT, last_name TEXT), age INT4)>, F FLOAT8); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_1.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_1.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_1.result new file mode 100644 index 0000000..fea61b5 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_1.result @@ -0,0 +1,38 @@ +{ + "IsExternal": false, + "TableName": "t1", + "Attributes": [ + { + "ColumnDefName": "a", + "DataTypeName": "TEXT", + "OpType": "DataType" + }, + { + "ColumnDefName": "b", + "DataTypeName": "INT4", + "OpType": "DataType" + }, + { + "ColumnDefName": "c", + "DataTypeName": "MAP", + "Map": { + "KeyType": { + "DataTypeName": "TEXT", + "OpType": "DataType" + }, + "ValueType": { + "DataTypeName": "INT8", + "OpType": "DataType" + } + }, + "OpType": "DataType" + }, + { + "ColumnDefName": "f", + "DataTypeName": "FLOAT8", + "OpType": "DataType" + } + ], + "IfNotExists": false, + "OpType": "CreateTable" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_2.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_2.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_2.result new file mode 100644 index 0000000..81c0b90 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_2.result @@ -0,0 +1,52 @@ +{ + "IsExternal": false, + "TableName": "t1", + "Attributes": [ + { + "ColumnDefName": "a", + "DataTypeName": "TEXT", + "OpType": "DataType" + }, + { + "ColumnDefName": "b", + "DataTypeName": "INT4", + "OpType": "DataType" + }, + { + "ColumnDefName": "c", + "DataTypeName": "MAP", + "Map": { + "KeyType": { + "DataTypeName": "TEXT", + "OpType": "DataType" + }, + "ValueType": { + "DataTypeName": "RECORD", + "Record": { + "Schema": [ + { + "ColumnDefName": "name", + "DataTypeName": "TEXT", + "OpType": "DataType" + }, + { + "ColumnDefName": "age", + "DataTypeName": "INT4", + "OpType": "DataType" + } + ] + }, + "OpType": "DataType" + } + }, + "OpType": "DataType" + }, + { + "ColumnDefName": "f", + "DataTypeName": "FLOAT8", + "OpType": "DataType" + } + ], + "IfNotExists": false, + "OpType": "CreateTable" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_3.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_3.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_3.result new file mode 100644 index 0000000..3259ffb --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_3.result @@ -0,0 +1,66 @@ +{ + "IsExternal": false, + "TableName": "t1", + "Attributes": [ + { + "ColumnDefName": "a", + "DataTypeName": "TEXT", + "OpType": "DataType" + }, + { + "ColumnDefName": "b", + "DataTypeName": "INT4", + "OpType": "DataType" + }, + { + "ColumnDefName": "c", + "DataTypeName": "MAP", + "Map": { + "KeyType": { + "DataTypeName": "TEXT", + "OpType": "DataType" + }, + "ValueType": { + "DataTypeName": "RECORD", + "Record": { + "Schema": [ + { + "ColumnDefName": "name", + "DataTypeName": "RECORD", + "Record": { + "Schema": [ + { + "ColumnDefName": "first_name", + "DataTypeName": "TEXT", + "OpType": "DataType" + }, + { + "ColumnDefName": "last_name", + "DataTypeName": "TEXT", + "OpType": "DataType" + } + ] + }, + "OpType": "DataType" + }, + { + "ColumnDefName": "age", + "DataTypeName": "INT4", + "OpType": "DataType" + } + ] + }, + "OpType": "DataType" + } + }, + "OpType": "DataType" + }, + { + "ColumnDefName": "f", + "DataTypeName": "FLOAT8", + "OpType": "DataType" + } + ], + "IfNotExists": false, + "OpType": "CreateTable" +} \ No newline at end of file
