Repository: tajo Updated Branches: refs/heads/master 6defbda96 -> f48d4bd04
TAJO-1329: Improve Schema class to support nested struct support. Closes #389 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/f48d4bd0 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/f48d4bd0 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/f48d4bd0 Branch: refs/heads/master Commit: f48d4bd04ec46b6678773209a3397555eb2ce24d Parents: 6defbda Author: Hyunsik Choi <[email protected]> Authored: Thu Mar 12 19:21:39 2015 -0700 Committer: Hyunsik Choi <[email protected]> Committed: Thu Mar 12 19:21:39 2015 -0700 ---------------------------------------------------------------------- CHANGES | 3 + .../apache/tajo/algebra/ColumnDefinition.java | 2 +- .../org/apache/tajo/algebra/DataTypeExpr.java | 18 ++- .../org/apache/tajo/catalog/CatalogUtil.java | 28 ++-- .../java/org/apache/tajo/catalog/Column.java | 45 ++++-- .../org/apache/tajo/catalog/DDLBuilder.java | 9 +- .../java/org/apache/tajo/catalog/Schema.java | 156 +++++++++++++------ .../org/apache/tajo/catalog/SchemaUtil.java | 42 +++++ .../java/org/apache/tajo/catalog/TypeDesc.java | 75 +++++++++ .../org/apache/tajo/catalog/TestSchema.java | 137 ++++++++++++++++ .../dictionary/ColumnsTableDescriptor.java | 7 +- .../tajo/catalog/store/AbstractDBStore.java | 64 +++++--- .../apache/tajo/catalog/store/MariaDBStore.java | 5 +- .../apache/tajo/catalog/store/MySQLStore.java | 11 +- .../apache/tajo/catalog/store/OracleStore.java | 2 +- .../tajo/catalog/store/PostgreSQLStore.java | 4 +- .../src/main/resources/schemas/derby/derby.xml | 12 +- .../main/resources/schemas/mariadb/columns.sql | 3 +- .../main/resources/schemas/mysql/columns.sql | 3 +- .../main/resources/schemas/oracle/oracle.xml | 10 +- .../resources/schemas/postgresql/postgresql.xml | 10 +- .../org/apache/tajo/catalog/TestCatalog.java | 101 ++++++++++++ .../cli/tsql/commands/DescTableCommand.java | 5 +- tajo-common/src/main/proto/DataTypes.proto | 10 ++ tajo-core/pom.xml | 56 +++++++ .../org/apache/tajo/cli/tools/TestTajoDump.java | 21 +++ .../org/apache/tajo/cli/tsql/TestTajoCli.java | 40 +++-- .../tajo/engine/query/TestCreateTable.java | 24 +++ .../TestCreateTable/testNestedRecord1.sql | 1 + .../TestCreateTable/testNestedRecord2.sql | 1 + .../TestSelectQuery/testExplainSelect.result | 4 +- .../testDescTableForNestedSchema.result | 29 ++++ .../results/TestTajoDump/testDump2.result | 16 ++ .../org/apache/tajo/plan/ExprAnnotator.java | 5 +- .../org/apache/tajo/plan/LogicalPlanner.java | 18 ++- .../org/apache/tajo/plan/TypeDeterminant.java | 4 +- 36 files changed, 828 insertions(+), 153 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 6854b4f..9bec78c 100644 --- a/CHANGES +++ b/CHANGES @@ -59,6 +59,9 @@ Release 0.11.0 - unreleased SUB TASKS + TAJO-1329: Improve Schema class to support nested struct support. + (hyunsik) + TAJO-1353: Nested record support in CREATE TABLE statement. (hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java index 80ecac4..f8ea0f1 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java @@ -44,7 +44,7 @@ public class ColumnDefinition extends DataTypeExpr { // nested records if (dataType.isNestedRecordType()) { - this.nestedRecord = dataType.nestedRecord; + this.nestedRecordTypes = dataType.nestedRecordTypes; } } http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java index 28f9a5e..b280397 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java @@ -21,7 +21,6 @@ package org.apache.tajo.algebra; import com.google.common.base.Objects; import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; -import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.util.TUtil; @@ -33,19 +32,18 @@ public class DataTypeExpr extends Expr { @Expose @SerializedName("Scale") Integer scale; @Expose @SerializedName("Record") - DataTypeExpr [] nestedRecord; + ColumnDefinition [] nestedRecordTypes; // not null if the type is RECORD public DataTypeExpr(String typeName) { super(OpType.DataType); this.typeName = typeName; } - public DataTypeExpr(DataTypeExpr [] nestedRecordTypes) { + public DataTypeExpr(ColumnDefinition [] nestedRecordTypes) { super(OpType.DataType); - // Please refer to DataTypes.proto. 'STRUCT' must be equivalent to Enum type in DataTypes.proto. - // STRUCT = 51; + // RECORD = 51 in DataTypes.proto this.typeName = Type.RECORD.name(); - this.nestedRecord = nestedRecordTypes; + this.nestedRecordTypes = nestedRecordTypes; } public String getTypeName() { @@ -56,6 +54,10 @@ public class DataTypeExpr extends Expr { return this.typeName.equals(Type.RECORD.name()); } + public ColumnDefinition [] getNestedRecordTypes() { + return nestedRecordTypes; + } + public boolean hasLengthOrPrecision() { return lengthOrPrecision != null; } @@ -91,7 +93,7 @@ public class DataTypeExpr extends Expr { return typeName.equals(another.typeName) && TUtil.checkEquals(lengthOrPrecision, another.lengthOrPrecision) && TUtil.checkEquals(scale, another.scale) && - TUtil.checkEquals(nestedRecord, another.nestedRecord); + TUtil.checkEquals(nestedRecordTypes, another.nestedRecordTypes); } @Override @@ -100,7 +102,7 @@ public class DataTypeExpr extends Expr { dataType.typeName = typeName; dataType.lengthOrPrecision = lengthOrPrecision; dataType.scale = scale; - dataType.nestedRecord = nestedRecord; + dataType.nestedRecordTypes = nestedRecordTypes; return dataType; } } http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 45609d0..afcff2d 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -318,8 +318,7 @@ public class CatalogUtil { } /** - * This method transforms the unqualified names of a given schema into - * the qualified names. + * This method transforms the unqualified names of a schema to the qualified names. * * @param tableName a table name to be prefixed * @param schema a schema to be transformed @@ -327,15 +326,9 @@ public class CatalogUtil { * @return */ public static SchemaProto getQualfiedSchema(String tableName, SchemaProto schema) { - SchemaProto.Builder revisedSchema = SchemaProto.newBuilder(schema); - revisedSchema.clearFields(); - for (ColumnProto col : schema.getFieldsList()) { - ColumnProto.Builder builder = ColumnProto.newBuilder(col); - builder.setName(tableName + CatalogConstants.IDENTIFIER_DELIMITER + extractSimpleName(col.getName())); - revisedSchema.addFields(builder.build()); - } - - return revisedSchema.build(); + Schema restored = new Schema(schema); + restored.setQualifier(tableName); + return restored.getProto(); } public static DataType newDataType(Type type, String code) { @@ -354,6 +347,19 @@ public class CatalogUtil { return DataType.newBuilder().setType(type).build(); } + /** + * Create a record type + * + * @param nestedFieldNum The number of nested fields + * @return RECORD DataType + */ + public static DataType newRecordType(int nestedFieldNum) { + DataType.Builder builder = DataType.newBuilder(); + builder.setType(Type.RECORD); + builder.setNumNestedFields(nestedFieldNum); + return builder.build(); + } + public static DataType [] newSimpleDataTypeArray(Type... types) { DataType [] dataTypes = new DataType[types.length]; for (int i = 0; i < types.length; i++) { http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java index aceb6f1..12edaa7 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java @@ -32,7 +32,18 @@ import org.apache.tajo.json.GsonObject; */ public class Column implements ProtoObject<ColumnProto>, GsonObject { @Expose protected String name; - @Expose protected DataType dataType; + @Expose protected TypeDesc typeDesc; + + /** + * Column Constructor + * + * @param name field name + * @param typeDesc Type description + */ + public Column(String name, TypeDesc typeDesc) { + this.name = name; + this.typeDesc = typeDesc; + } /** * @@ -40,8 +51,7 @@ public class Column implements ProtoObject<ColumnProto>, GsonObject { * @param dataType Data Type with length */ public Column(String name, DataType dataType) { - this.name = name; - this.dataType = dataType; + this(name, new TypeDesc(dataType)); } /** @@ -65,7 +75,7 @@ public class Column implements ProtoObject<ColumnProto>, GsonObject { public Column(ColumnProto proto) { name = proto.getName(); - dataType = proto.getDataType(); + typeDesc = new TypeDesc(proto.getDataType()); } /** @@ -101,24 +111,33 @@ public class Column implements ProtoObject<ColumnProto>, GsonObject { } /** + * Return type description + * + * @return TypeDesc + */ + public TypeDesc getTypeDesc() { + return this.typeDesc; + } + + /** * * @return DataType which includes domain type and scale. */ public DataType getDataType() { - return this.dataType; + return this.typeDesc.dataType; } @Override public boolean equals(Object o) { if (o instanceof Column) { Column another = (Column)o; - return name.equals(another.name) && dataType.equals(another.dataType); + return name.equals(another.name) && typeDesc.equals(another.typeDesc); } return false; } public int hashCode() { - return Objects.hashCode(name, dataType); + return Objects.hashCode(name, typeDesc); } @@ -128,16 +147,16 @@ public class Column implements ProtoObject<ColumnProto>, GsonObject { */ @Override public ColumnProto getProto() { - return ColumnProto.newBuilder().setName(this.name).setDataType(this.dataType).build(); + ColumnProto.Builder builder = ColumnProto.newBuilder(); + builder + .setName(this.name) + .setDataType(this.typeDesc.getDataType()); + return builder.build(); } public String toString() { StringBuilder sb = new StringBuilder(getQualifiedName()); - sb.append(" (").append(getDataType().getType()); - if (getDataType().getLength() > 0) { - sb.append("(" + getDataType().getLength() + ")"); - } - sb.append(")"); + sb.append(" (").append(typeDesc.toString()).append(")"); return sb.toString(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java index 978092a..1a59e88 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java @@ -69,7 +69,7 @@ public class DDLBuilder { return sb.toString(); } - private static void buildSchema(StringBuilder sb, Schema schema) { + public static void buildSchema(StringBuilder sb, Schema schema) { boolean first = true; sb.append(" ("); @@ -81,11 +81,8 @@ public class DDLBuilder { } sb.append(CatalogUtil.denormalizeIdentifier(column.getSimpleName())).append(" "); - TajoDataTypes.DataType dataType = column.getDataType(); - sb.append(dataType.getType().name()); - if (column.getDataType().hasLength() && column.getDataType().getLength() > 0) { - sb.append(" (").append(column.getDataType().getLength()).append(")"); - } + TypeDesc typeDesc = column.getTypeDesc(); + sb.append(typeDesc); } sb.append(")"); } http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java index 71c1b01..ed2cd2c 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList; import com.google.gson.annotations.Expose; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.tajo.catalog.SchemaUtil.ColumnVisitor; import org.apache.tajo.catalog.exception.AlreadyExistsFieldException; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto; @@ -45,36 +46,77 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject { public Schema() { init(); } - + + /** + * This Schema constructor restores a serialized schema into in-memory Schema structure. + * A serialized schema is an ordered list in depth-first order over a nested schema. + * This constructor transforms the list into a tree-like structure. + * + * @param proto + */ public Schema(SchemaProto proto) { - this.fields = new ArrayList<Column>(); - this.fieldsByQualifiedName = new HashMap<String, Integer>(); - this.fieldsByName = new HashMap<String, List<Integer>>(); - for(ColumnProto colProto : proto.getFieldsList()) { - Column tobeAdded = new Column(colProto); - fields.add(tobeAdded); - if (tobeAdded.hasQualifier()) { - fieldsByQualifiedName.put(tobeAdded.getQualifier() + "." + tobeAdded.getSimpleName(), fields.size() - 1); - } else { - fieldsByQualifiedName.put(tobeAdded.getSimpleName(), fields.size() - 1); - } - if (fieldsByName.containsKey(tobeAdded.getSimpleName())) { - fieldsByName.get(tobeAdded.getSimpleName()).add(fields.size() - 1); - } else { - fieldsByName.put(tobeAdded.getSimpleName(), TUtil.newList(fields.size() - 1)); + init(); + + List<Column> toBeAdded = TUtil.newList(); + for (int i = 0; i < proto.getFieldsCount(); i++) { + deserializeColumn(toBeAdded, proto.getFieldsList(), i); + } + + for (Column c : toBeAdded) { + addColumn(c); + } + } + + /** + * This method transforms a list of ColumnProtos into a schema tree. + * It assumes that <code>protos</code> contains a list of ColumnProtos in the depth-first order. + * + * @param tobeAdded + * @param protos + * @param serializedColumnIndex + */ + private static void deserializeColumn(List<Column> tobeAdded, List<ColumnProto> protos, int serializedColumnIndex) { + ColumnProto columnProto = protos.get(serializedColumnIndex); + if (columnProto.getDataType().getType() == Type.RECORD) { + + // Get the number of child fields + int childNum = columnProto.getDataType().getNumNestedFields(); + // where is start index of nested fields? + int childStartIndex = tobeAdded.size() - childNum; + // Extract nested fields + List<Column> nestedColumns = TUtil.newList(tobeAdded.subList(childStartIndex, childStartIndex + childNum)); + + // Remove nested fields from the the current level + for (int i = 0; i < childNum; i++) { + tobeAdded.remove(tobeAdded.size() - 1); } + + // Add the nested fields to the list as a single record column + tobeAdded.add(new Column(columnProto.getName(), new TypeDesc(new Schema(nestedColumns)))); + } else { + tobeAdded.add(new Column(protos.get(serializedColumnIndex))); } } public Schema(Schema schema) { this(); + this.fields.addAll(schema.fields); this.fieldsByQualifiedName.putAll(schema.fieldsByQualifiedName); this.fieldsByName.putAll(schema.fieldsByName); } - public Schema(Column [] columns) { + public Schema(Column [] columns) { init(); + + for(Column c : columns) { + addColumn(c); + } + } + + public Schema(Iterable<Column> columns) { + init(); + for(Column c : columns) { addColumn(c); } @@ -93,21 +135,15 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject { * @param qualifier The qualifier */ public void setQualifier(String qualifier) { - Schema copy = null; - try { - copy = (Schema) clone(); - } catch (CloneNotSupportedException e) { - throw new RuntimeException(e); - } + List<Column> columns = getColumns(); fields.clear(); fieldsByQualifiedName.clear(); fieldsByName.clear(); Column newColumn; - for (int i = 0; i < copy.size(); i++) { - Column column = copy.getColumn(i); - newColumn = new Column(qualifier + "." + column.getSimpleName(), column.getDataType()); + for (Column c : columns) { + newColumn = new Column(qualifier + "." + c.getSimpleName(), c.typeDesc); addColumn(newColumn); } } @@ -275,6 +311,21 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject { return fields.containsAll(columns); } + public synchronized Schema addColumn(String name, TypeDesc typeDesc) { + String normalized = name; + if(fieldsByQualifiedName.containsKey(normalized)) { + LOG.error("Already exists column " + normalized); + throw new AlreadyExistsFieldException(normalized); + } + + Column newCol = new Column(normalized, typeDesc); + fields.add(newCol); + fieldsByQualifiedName.put(newCol.getQualifiedName(), fields.size() - 1); + fieldsByName.put(newCol.getSimpleName(), TUtil.newList(fields.size() - 1)); + + return this; + } + public synchronized Schema addColumn(String name, Type type) { if (type == Type.CHAR) { return addColumn(name, CatalogUtil.newDataTypeWithLen(type, 1)); @@ -287,22 +338,13 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject { } public synchronized Schema addColumn(String name, DataType dataType) { - String normalized = name; - if(fieldsByQualifiedName.containsKey(normalized)) { - LOG.error("Already exists column " + normalized); - throw new AlreadyExistsFieldException(normalized); - } - - Column newCol = new Column(normalized, dataType); - fields.add(newCol); - fieldsByQualifiedName.put(newCol.getQualifiedName(), fields.size() - 1); - fieldsByName.put(newCol.getSimpleName(), TUtil.newList(fields.size() - 1)); - + addColumn(name, new TypeDesc(dataType)); + return this; } public synchronized void addColumn(Column column) { - addColumn(column.getQualifiedName(), column.getDataType()); + addColumn(column.getQualifiedName(), column.typeDesc); } public synchronized void addColumns(Schema schema) { @@ -327,10 +369,9 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject { @Override public Object clone() throws CloneNotSupportedException { - Schema schema = null; - - schema = (Schema) super.clone(); + Schema schema = (Schema) super.clone(); schema.init(); + for(Column column: this.fields) { schema.addColumn(column); } @@ -340,15 +381,34 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject { @Override public SchemaProto getProto() { SchemaProto.Builder builder = SchemaProto.newBuilder(); - builder.clearFields(); - if (this.fields != null) { - for(Column col : fields) { - builder.addFields(col.getProto()); - } - } + SchemaProtoBuilder recursiveBuilder = new SchemaProtoBuilder(builder); + SchemaUtil.visitSchema(this, recursiveBuilder); return builder.build(); } + private static class SchemaProtoBuilder implements ColumnVisitor { + private SchemaProto.Builder builder; + public SchemaProtoBuilder(SchemaProto.Builder builder) { + this.builder = builder; + } + + @Override + public void visit(int depth, Column column) { + + if (column.getDataType().getType() == Type.RECORD) { + DataType.Builder updatedType = DataType.newBuilder(column.getDataType()); + updatedType.setNumNestedFields(column.typeDesc.nestedRecordSchema.size()); + + ColumnProto.Builder updatedColumn = ColumnProto.newBuilder(column.getProto()); + updatedColumn.setDataType(updatedType); + + builder.addFields(updatedColumn.build()); + } else { + builder.addFields(column.getProto()); + } + } + } + public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{(").append(size()).append(") "); @@ -356,7 +416,7 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject { for(Column col : fields) { sb.append(col); if (i < fields.size() - 1) { - sb.append(","); + sb.append(", "); } i++; } http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java index 23ebe1b..f2bb71c 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java @@ -108,4 +108,46 @@ public class SchemaUtil { } return names; } + + /** + * Column visitor interface + */ + public static interface ColumnVisitor { + public void visit(int depth, Column column); + } + + /** + * It allows a column visitor to traverse all columns in a schema in a depth-first order. + * @param schema + * @param function + */ + public static void visitSchema(Schema schema, ColumnVisitor function) { + for(Column col : schema.getColumns()) { + visitInDepthFirstOrder(0, function, col); + } + } + + /** + * A recursive function to traverse all columns in a schema in a depth-first order. + * + * @param depth Nested depth. 0 is root column. + * @param function Visitor + * @param column Current visiting column + */ + private static void visitInDepthFirstOrder(int depth, ColumnVisitor function, Column column) { + if (column.getDataType().getType() == Type.RECORD) { + for (Column nestedColumn : column.typeDesc.nestedRecordSchema.getColumns()) { + visitInDepthFirstOrder(depth + 1, function, nestedColumn); + } + function.visit(depth, column); + } else { + function.visit(depth, column); + } + } + + public static String toDisplayString(Schema schema) { + StringBuilder sb = new StringBuilder(); + DDLBuilder.buildSchema(sb, schema); + return sb.toString(); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeDesc.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeDesc.java new file mode 100644 index 0000000..3bd0f00 --- /dev/null +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeDesc.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog; + +import com.google.common.base.Objects; +import com.google.gson.annotations.Expose; +import org.apache.tajo.common.TajoDataTypes.DataType; +import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.util.TUtil; + +/** + * Type Description for a column + */ +public class TypeDesc { + @Expose protected DataType dataType; + @Expose protected Schema nestedRecordSchema; // NULL unless type is RECORD. + + public TypeDesc(DataType dataType) { + this.dataType = dataType; + } + + public TypeDesc(Schema recordSchema) { + this.dataType = CatalogUtil.newSimpleDataType(Type.RECORD); + this.nestedRecordSchema = recordSchema; + } + + public DataType getDataType() { + return dataType; + } + + public boolean equals(Object obj) { + if (obj instanceof TypeDesc) { + TypeDesc other = (TypeDesc) obj; + return this.dataType.equals(other.dataType) && + TUtil.checkEquals(nestedRecordSchema, other.nestedRecordSchema); + + } else { + return false; + } + } + + public int hashCode() { + return Objects.hashCode(dataType.hashCode(), nestedRecordSchema); + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + + if (dataType.getType() == Type.RECORD) { + sb.append("RECORD").append(SchemaUtil.toDisplayString(nestedRecordSchema)).append(""); + } else { + sb.append(dataType.getType().name()); + if (dataType.getLength() > 0) { + sb.append("(" + dataType.getLength() + ")"); + } + } + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java index a61b422..edd0f3e 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java @@ -34,6 +34,87 @@ public class TestSchema { Column col2; Column col3; + public static final Schema nestedSchema1; + public static final Schema nestedSchema2; + public static final Schema nestedSchema3; + + static { + // simple nested schema + nestedSchema1 = new Schema(); + nestedSchema1.addColumn("s1", Type.INT8); + + Schema nestedRecordSchema = new Schema(); + nestedRecordSchema.addColumn("s2", Type.FLOAT4); + nestedRecordSchema.addColumn("s3", Type.TEXT); + + Column nestedField = new Column("s4", new TypeDesc(nestedRecordSchema)); + nestedSchema1.addColumn(nestedField); + + nestedSchema1.addColumn("s5", Type.FLOAT8); + + // two level nested schema + // + // s1 + // |- s2 + // |- s4 + // |- s3 + // |- s4 + // |- s5 + // |- s8 + // |- s6 + // |- s7 + nestedSchema2 = new Schema(); + nestedSchema2.addColumn("s1", Type.INT8); + + Schema nestedRecordSchema1 = new Schema(); + nestedRecordSchema1.addColumn("s2", Type.FLOAT4); + nestedRecordSchema1.addColumn("s3", Type.TEXT); + + Column nestedField1 = new Column("s4", new TypeDesc(nestedRecordSchema1)); + nestedSchema2.addColumn(nestedField1); + + nestedSchema2.addColumn("s5", Type.FLOAT8); + + Schema nestedRecordSchema2 = new Schema(); + nestedRecordSchema2.addColumn("s6", Type.FLOAT4); + nestedRecordSchema2.addColumn("s7", Type.TEXT); + + Column nestedField2 = new Column("s8", new TypeDesc(nestedRecordSchema2)); + nestedSchema2.addColumn(nestedField2); + + + // three level nested schema + // + // s1 + // |- s2 + // |- s3 + // |- s4 + // |- s7 + // |- s5 + // |- s6 + // |- s8 + // |- s9 + + nestedSchema3 = new Schema(); + nestedSchema3.addColumn("s1", Type.INT8); + + nestedSchema3.addColumn("s2", Type.INT8); + + Schema s5 = new Schema(); + s5.addColumn("s6", Type.INT8); + + Schema s7 = new Schema(); + s7.addColumn("s5", new TypeDesc(s5)); + + Schema s3 = new Schema(); + s3.addColumn("s4", Type.INT8); + s3.addColumn("s7", new TypeDesc(s7)); + s3.addColumn("s8", Type.INT8); + + nestedSchema3.addColumn("s3", new TypeDesc(s3)); + nestedSchema3.addColumn("s9", Type.INT8); + } + @Before public void setUp() throws Exception { schema = new Schema(); @@ -167,6 +248,62 @@ public class TestSchema { assertEquals(schema3.getColumn(0), schema3.getColumn("tb2.col1")); assertEquals(schema3.getColumn(1), schema3.getColumn("col2")); assertEquals(schema3.getColumn(1), schema3.getColumn("tb2.col2")); + } + + @Test + public void testNestedRecord1() { + verifySchema(nestedSchema1); + } + + @Test + public void testNestedRecord2() { + verifySchema(nestedSchema2); + } + + @Test + public void testNestedRecord3() { + verifySchema(nestedSchema3); + } + + @Test + public void testNestedRecord4() { + Schema root = new Schema(); + + Schema nf2DotNf1 = new Schema(); + nf2DotNf1.addColumn("f1", Type.INT8); + nf2DotNf1.addColumn("f2", Type.INT8); + + Schema nf2DotNf2 = new Schema(); + nf2DotNf2.addColumn("f1", Type.INT8); + nf2DotNf2.addColumn("f2", Type.INT8); + + Schema nf2 = new Schema(); + nf2.addColumn("f1", Type.INT8); + nf2.addColumn("nf1", new TypeDesc(nf2DotNf1)); + nf2.addColumn("nf2", new TypeDesc(nf2DotNf2)); + nf2.addColumn("f2", Type.INT8); + + root.addColumn("f1", Type.INT8); + root.addColumn("nf1", Type.INT8); + root.addColumn("nf2", new TypeDesc(nf2)); + root.addColumn("f2", Type.INT8); + + verifySchema(root); + } + + public static void verifySchema(Schema s1) { + assertEquals(s1, s1); + + SchemaProto proto = s1.getProto(); + assertEquals("Proto (de)serialized schema is different from the original: ", s1, new Schema(proto)); + + Schema cloned = null; + try { + cloned = (Schema) s1.clone(); + } catch (CloneNotSupportedException e) { + fail("Clone is failed"); + } + assertEquals("Cloned schema is different from the original one:", s1, cloned); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/ColumnsTableDescriptor.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/ColumnsTableDescriptor.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/ColumnsTableDescriptor.java index 85b8f20..9ab65ec 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/ColumnsTableDescriptor.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/ColumnsTableDescriptor.java @@ -24,9 +24,10 @@ class ColumnsTableDescriptor extends AbstractTableDescriptor { private static final String TABLENAME = "columns"; private final ColumnDescriptor[] columns = new ColumnDescriptor[] { - new ColumnDescriptor("tid", Type.INT4, 0), - new ColumnDescriptor("column_name", Type.TEXT, 0), - new ColumnDescriptor("ordinal_position", Type.INT4, 0), + new ColumnDescriptor("tid", Type.INT4, 0), // just key for DBMS + new ColumnDescriptor("column_name", Type.TEXT, 0), // column name + new ColumnDescriptor("ordinal_position", Type.INT4, 0), // the ordinal position in a schema + new ColumnDescriptor("nested_field_num", Type.INT4, 0), // the number of child nested fields new ColumnDescriptor("data_type", Type.TEXT, 0), new ColumnDescriptor("type_length", Type.INT4, 0) }; http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 5c78f82..be6bf1c 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -33,6 +33,7 @@ import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.exception.*; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.*; +import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.exception.InternalException; import org.apache.tajo.exception.UnimplementedException; @@ -800,7 +801,10 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo pstmt.close(); String colSql = - "INSERT INTO " + TB_COLUMNS + " (TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, TYPE_LENGTH) VALUES(?, ?, ?, ?, ?) "; + "INSERT INTO " + TB_COLUMNS + + // 1 2 3 4 5 6 + " (TID, COLUMN_NAME, ORDINAL_POSITION, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH)" + + " VALUES(?, ?, ?, ?, ?, ?) "; if (LOG.isDebugEnabled()) { LOG.debug(colSql); @@ -809,11 +813,15 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo pstmt = conn.prepareStatement(colSql); for (int i = 0; i < table.getSchema().getFieldsCount(); i++) { ColumnProto col = table.getSchema().getFields(i); + TajoDataTypes.DataType dataType = col.getDataType(); + pstmt.setInt(1, tableId); pstmt.setString(2, CatalogUtil.extractSimpleName(col.getName())); pstmt.setInt(3, i); - pstmt.setString(4, col.getDataType().getType().name()); - pstmt.setInt(5, (col.getDataType().hasLength() ? col.getDataType().getLength() : 0)); + // the default number of nested fields is 0. + pstmt.setInt(4, dataType.hasNumNestedFields() ? dataType.getNumNestedFields() : 0); + pstmt.setString(5, dataType.getType().name()); + pstmt.setInt(6, (col.getDataType().hasLength() ? col.getDataType().getLength() : 0)); pstmt.addBatch(); pstmt.clearParameters(); } @@ -1028,13 +1036,13 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo throws CatalogException { final String selectColumnSql = - "SELECT COLUMN_NAME, DATA_TYPE, TYPE_LENGTH, ORDINAL_POSITION from " + TB_COLUMNS + + "SELECT COLUMN_NAME, DATA_TYPE, TYPE_LENGTH, ORDINAL_POSITION, NESTED_FIELD_NUM from " + TB_COLUMNS + " WHERE " + COL_TABLES_PK + " = ?" + " AND COLUMN_NAME = ?" ; final String deleteColumnNameSql = "DELETE FROM " + TB_COLUMNS + " WHERE TID = ? AND COLUMN_NAME = ?"; final String insertNewColumnSql = "INSERT INTO " + TB_COLUMNS + - " (TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, TYPE_LENGTH) VALUES(?, ?, ?, ?, ?) "; + " (TID, COLUMN_NAME, ORDINAL_POSITION, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH) VALUES(?, ?, ?, ?, ?, ?) "; if (LOG.isDebugEnabled()) { LOG.debug(selectColumnSql); @@ -1058,13 +1066,15 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo resultSet = pstmt.executeQuery(); CatalogProtos.ColumnProto columnProto = null; - int ordinalPosition = -1; + int ordinalPosition = 0; + int nestedFieldNum = 0; if (resultSet.next()) { columnProto = resultToColumnProto(resultSet); //NOTE ==> Setting new column Name columnProto = columnProto.toBuilder().setName(alterColumnProto.getNewColumnName()).build(); ordinalPosition = resultSet.getInt("ORDINAL_POSITION"); + nestedFieldNum = resultSet.getInt("NESTED_FIELD_NUM"); } else { throw new NoSuchColumnException(alterColumnProto.getOldColumnName()); } @@ -1085,8 +1095,9 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo pstmt.setInt(1, tableId); pstmt.setString(2, CatalogUtil.extractSimpleName(columnProto.getName())); pstmt.setInt(3, ordinalPosition); - pstmt.setString(4, columnProto.getDataType().getType().name()); - pstmt.setInt(5, (columnProto.getDataType().hasLength() ? columnProto.getDataType().getLength() : 0)); + pstmt.setInt(4, nestedFieldNum); + pstmt.setString(5, columnProto.getDataType().getType().name()); + pstmt.setInt(6, (columnProto.getDataType().hasLength() ? columnProto.getDataType().getLength() : 0)); pstmt.executeUpdate(); conn.commit(); @@ -1101,8 +1112,11 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo private void addNewColumn(int tableId, CatalogProtos.ColumnProto columnProto) throws CatalogException { - final String insertNewColumnSql = "INSERT INTO " + TB_COLUMNS + " (TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, TYPE_LENGTH) VALUES(?, ?, ?, ?, ?) "; - final String columnCountSql = "SELECT COLUMN_NAME, MAX(ORDINAL_POSITION) AS POSITION FROM " + TB_COLUMNS + " WHERE TID = ? GROUP BY COLUMN_NAME"; + final String insertNewColumnSql = + "INSERT INTO " + TB_COLUMNS + + " (TID, COLUMN_NAME, ORDINAL_POSITION, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH) VALUES(?, ?, ?, ?, ?, ?) "; + final String columnCountSql = + "SELECT MAX(ORDINAL_POSITION) AS POSITION FROM " + TB_COLUMNS + " WHERE TID = ?"; if (LOG.isDebugEnabled()) { LOG.debug(insertNewColumnSql); @@ -1119,18 +1133,22 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo pstmt.setInt(1 , tableId); resultSet = pstmt.executeQuery(); + // get the last the ordinal position. int position = resultSet.next() ? resultSet.getInt("POSITION") : 0; resultSet.close(); pstmt.close(); resultSet = null; + TajoDataTypes.DataType dataType = columnProto.getDataType(); + pstmt = conn.prepareStatement(insertNewColumnSql); pstmt.setInt(1, tableId); pstmt.setString(2, CatalogUtil.extractSimpleName(columnProto.getName())); pstmt.setInt(3, position + 1); - pstmt.setString(4, columnProto.getDataType().getType().name()); - pstmt.setInt(5, (columnProto.getDataType().hasLength() ? columnProto.getDataType().getLength() : 0)); + pstmt.setInt(4, dataType.hasNumNestedFields() ? dataType.getNumNestedFields() : 0); + pstmt.setString(5, dataType.getType().name()); + pstmt.setInt(6, (columnProto.getDataType().hasLength() ? columnProto.getDataType().getLength() : 0)); pstmt.executeUpdate(); } catch (SQLException sqlException) { @@ -1385,7 +1403,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo // Geting Column Descriptions ////////////////////////////////////////// CatalogProtos.SchemaProto.Builder schemaBuilder = CatalogProtos.SchemaProto.newBuilder(); - sql = "SELECT COLUMN_NAME, DATA_TYPE, TYPE_LENGTH from " + TB_COLUMNS + + sql = "SELECT COLUMN_NAME, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH from " + TB_COLUMNS + " WHERE " + COL_TABLES_PK + " = ? ORDER BY ORDINAL_POSITION ASC"; if (LOG.isDebugEnabled()) { @@ -1643,7 +1661,8 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo List<ColumnProto> columns = new ArrayList<ColumnProto>(); try { - String sql = "SELECT TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, TYPE_LENGTH FROM " + TB_COLUMNS + + String sql = + "SELECT TID, COLUMN_NAME, ORDINAL_POSITION, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH FROM " + TB_COLUMNS + " ORDER BY TID ASC, ORDINAL_POSITION ASC"; conn = getConnection(); @@ -1654,11 +1673,15 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo builder.setTid(resultSet.getInt("TID")); builder.setName(resultSet.getString("COLUMN_NAME")); - + + int nestedFieldNum = resultSet.getInt("NESTED_FIELD_NUM"); + Type type = getDataType(resultSet.getString("DATA_TYPE").trim()); int typeLength = resultSet.getInt("TYPE_LENGTH"); - - if (typeLength > 0) { + + if (nestedFieldNum > 0) { + builder.setDataType(CatalogUtil.newRecordType(nestedFieldNum)); + } else if (typeLength > 0) { builder.setDataType(CatalogUtil.newDataTypeWithLen(type, typeLength)); } else { builder.setDataType(CatalogUtil.newSimpleDataType(type)); @@ -2309,9 +2332,14 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo ColumnProto.Builder builder = ColumnProto.newBuilder(); builder.setName(res.getString("column_name").trim()); + int nestedFieldNum = res.getInt("NESTED_FIELD_NUM"); + Type type = getDataType(res.getString("data_type").trim()); int typeLength = res.getInt("type_length"); - if (typeLength > 0) { + + if (nestedFieldNum > 0) { + builder.setDataType(CatalogUtil.newRecordType(nestedFieldNum)); + } else if (typeLength > 0) { builder.setDataType(CatalogUtil.newDataTypeWithLen(type, typeLength)); } else { builder.setDataType(CatalogUtil.newSimpleDataType(type)); http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MariaDBStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MariaDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MariaDBStore.java index 8cb3858..0159b31 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MariaDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MariaDBStore.java @@ -26,8 +26,9 @@ import org.apache.tajo.catalog.exception.CatalogException; import org.apache.tajo.exception.InternalException; public class MariaDBStore extends AbstractMySQLMariaDBStore { - /** 2014-06-09: First versioning */ - private static final int MARIADB_CATALOG_STORE_VERSION = 2; + /** 3 - 2015-03-12: Nested Schema (TAJO-1329) */ + /** 2 - First versioning */ + private static final int MARIADB_CATALOG_STORE_VERSION = 3; private static final String CATALOG_DRIVER = "org.mariadb.jdbc.Driver"; http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MySQLStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MySQLStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MySQLStore.java index cedc0fe..f7f1b1b 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MySQLStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MySQLStore.java @@ -26,10 +26,11 @@ import org.apache.tajo.catalog.exception.CatalogException; import org.apache.tajo.exception.InternalException; public class MySQLStore extends AbstractMySQLMariaDBStore { - /** 2014-03-20: First versioning */ - private static final int MYSQL_CATALOG_STORE_VERSION_2 = 2; - /** Before 2013-03-20 */ - private static final int MYSQL_CATALOG_STORE_VERSION_1 = 1; + + /** 3 - 2015-03-12: Nested Schema (TAJO-1329) */ + /** 2 - 2014-06-09: First versioning */ + /** 1- Before 2013-03-20 */ + private static final int MYSQL_CATALOG_STORE_VERSION = 3; private static final String CATALOG_DRIVER = "com.mysql.jdbc.Driver"; @Override @@ -43,7 +44,7 @@ public class MySQLStore extends AbstractMySQLMariaDBStore { @Override public int getDriverVersion() { - return MYSQL_CATALOG_STORE_VERSION_2; + return MYSQL_CATALOG_STORE_VERSION; } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/OracleStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/OracleStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/OracleStore.java index 45c153c..4b7e6a3 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/OracleStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/OracleStore.java @@ -27,7 +27,7 @@ import org.apache.tajo.catalog.exception.CatalogException; import org.apache.tajo.exception.InternalException; public class OracleStore extends AbstractDBStore { - + private static final String CATALOG_DRIVER = "oracle.jdbc.OracleDriver"; public OracleStore(Configuration conf) throws InternalException { http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/PostgreSQLStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/PostgreSQLStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/PostgreSQLStore.java index 41f2909..6089fdd 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/PostgreSQLStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/PostgreSQLStore.java @@ -27,7 +27,7 @@ import org.apache.tajo.catalog.exception.CatalogException; import org.apache.tajo.exception.InternalException; public class PostgreSQLStore extends AbstractDBStore { - + private static final String CATALOG_DRIVER = "org.postgresql.Driver"; public PostgreSQLStore(Configuration conf) throws InternalException { @@ -48,7 +48,7 @@ public class PostgreSQLStore extends AbstractDBStore { protected Connection createConnection(Configuration conf) throws SQLException { return DriverManager.getConnection(getCatalogUri(), this.connectionId, this.connectionPassword); } - + @Override protected void createDatabaseDependants() throws CatalogException { http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml index db2473b..a0bd9cd 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml @@ -17,7 +17,12 @@ limitations under the License. --> <tns:store xmlns:tns="http://tajo.apache.org/catalogstore"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd "> - <tns:base version="2"> + + <!-- Version History --> + <!-- 3 - 2015-03-12: Nested Schema (TAJO-1329) --> + <!-- 2 - First versioning --> + + <tns:base version="3"> <tns:objects> <tns:Object order="0" type="table" name="META"> <tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql> @@ -72,14 +77,15 @@ TID INT NOT NULL REFERENCES TABLES (TID) ON DELETE CASCADE, COLUMN_NAME VARCHAR(128) NOT NULL, ORDINAL_POSITION INTEGER NOT NULL, + NESTED_FIELD_NUM INTEGER NOT NULL, DATA_TYPE CHAR(16), TYPE_LENGTH INTEGER, - CONSTRAINT COLUMNS_PK PRIMARY KEY (TID, COLUMN_NAME) + CONSTRAINT COLUMNS_PK PRIMARY KEY (TID, ORDINAL_POSITION) )]]> </tns:sql> </tns:Object> <tns:Object order="8" type="index" name="IDX_FK_COLUMNS_TABLE_NAME" dependsOn="COLUMNS"> - <tns:sql><![CDATA[CREATE UNIQUE INDEX idx_fk_columns_table_name on COLUMNS (TID, COLUMN_NAME)]]></tns:sql> + <tns:sql><![CDATA[CREATE UNIQUE INDEX idx_fk_columns_table_name on COLUMNS (TID, ORDINAL_POSITION)]]></tns:sql> </tns:Object> <tns:Object order="9" type="table" name="OPTIONS"> <tns:sql><![CDATA[ http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/columns.sql ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/columns.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/columns.sql index aec1553..5b8ed26 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/columns.sql +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/columns.sql @@ -2,8 +2,9 @@ CREATE TABLE COLUMNS ( TID INT NOT NULL, COLUMN_NAME VARCHAR(255) BINARY NOT NULL, ORDINAL_POSITION INT NOT NULL, + NESTED_FIELD_NUM INT NOT NULL, DATA_TYPE CHAR(16), TYPE_LENGTH INTEGER, - PRIMARY KEY (TID, COLUMN_NAME), + PRIMARY KEY (TID, ORDINAL_POSITION), FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE ) http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/columns.sql ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/columns.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/columns.sql index aec1553..5b8ed26 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/columns.sql +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/columns.sql @@ -2,8 +2,9 @@ CREATE TABLE COLUMNS ( TID INT NOT NULL, COLUMN_NAME VARCHAR(255) BINARY NOT NULL, ORDINAL_POSITION INT NOT NULL, + NESTED_FIELD_NUM INT NOT NULL, DATA_TYPE CHAR(16), TYPE_LENGTH INTEGER, - PRIMARY KEY (TID, COLUMN_NAME), + PRIMARY KEY (TID, ORDINAL_POSITION), FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE ) http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml index 8945fca..880a14e 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml @@ -17,7 +17,12 @@ limitations under the License. --> <tns:store xmlns:tns="http://tajo.apache.org/catalogstore"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd "> - <tns:base version="2"> + + <!-- Version History --> + <!-- 3 - 2015-03-12: Nested Schema (TAJO-1329) --> + <!-- 2 - First versioning --> + + <tns:base version="3"> <tns:objects> <tns:Object order="0" type="table" name="meta"> <tns:sql><![CDATA[ @@ -117,9 +122,10 @@ TID INT NOT NULL, COLUMN_NAME VARCHAR2(255) NOT NULL, ORDINAL_POSITION INT NOT NULL, + NESTED_FIELD_NUM INT NOT NULL, DATA_TYPE CHAR(16), TYPE_LENGTH INTEGER, - CONSTRAINT COLUMNS_PKEY PRIMARY KEY (TID, COLUMN_NAME), + CONSTRAINT COLUMNS_PKEY PRIMARY KEY (TID, ORDINAL_POSITION), FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE )]]> </tns:sql> http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml index 8e5cbcc..821527b 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml @@ -19,7 +19,12 @@ <tns:store xmlns:tns="http://tajo.apache.org/catalogstore"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd "> - <tns:base version="2"> + + <!-- Version History --> + <!-- 3 - 2015-03-12: Nested Schema (TAJO-1329) --> + <!-- 2 - First versioning --> + + <tns:base version="3"> <tns:objects> <tns:Object name="META" type="table" order="0"> <tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql> @@ -76,9 +81,10 @@ xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition. TID INT NOT NULL, COLUMN_NAME VARCHAR(255) NOT NULL, ORDINAL_POSITION INT NOT NULL, + NESTED_FIELD_NUM INT NOT NULL, DATA_TYPE CHAR(16), TYPE_LENGTH INTEGER, - CONSTRAINT COLUMNS_PKEY PRIMARY KEY (TID, COLUMN_NAME), + CONSTRAINT COLUMNS_PKEY PRIMARY KEY (TID, ORDINAL_POSITION), FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE )]]> </tns:sql> http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 305742f..c3bfc99 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -431,6 +431,107 @@ public class TestCatalog { assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, "getTable")); } + /** + * It asserts the equality between an original table desc and a restored table desc. + */ + private static void assertSchemaEquality(String tableName, Schema schema) throws IOException { + Path path = new Path(CommonTestingUtil.getTestDir(), tableName); + TableDesc tableDesc = new TableDesc( + CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName), + schema, + StoreType.CSV, + new KeyValueSet(), + path.toUri()); + + // schema creation + assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); + catalog.createTable(tableDesc); + assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); + + // change it for the equals test. + schema.setQualifier(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName)); + TableDesc restored = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); + assertEquals(schema, restored.getSchema()); + + // drop test + catalog.dropTable(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName)); + assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); + } + + @Test + public void testCreateAndGetNestedTable1() throws Exception { + // schema creation + // three level nested schema + // + // s1 + // |- s2 + // |- s3 + // |- s4 + // |- s7 + // |- s5 + // |- s6 + // |- s8 + // |- s9 + + Schema nestedSchema = new Schema(); + nestedSchema.addColumn("s1", Type.INT8); + + nestedSchema.addColumn("s2", Type.INT8); + + Schema s5 = new Schema(); + s5.addColumn("s6", Type.INT8); + + Schema s7 = new Schema(); + s7.addColumn("s5", new TypeDesc(s5)); + + Schema s3 = new Schema(); + s3.addColumn("s4", Type.INT8); + s3.addColumn("s7", new TypeDesc(s7)); + s3.addColumn("s8", Type.INT8); + + nestedSchema.addColumn("s3", new TypeDesc(s3)); + nestedSchema.addColumn("s9", Type.INT8); + + assertSchemaEquality("nested_schema1", nestedSchema); + } + + @Test + public void testCreateAndGetNestedTable2() throws Exception { + // schema creation + // three level nested schema + // + // s1 + // |- s2 + // |- s3 + // |- s1 + // |- s2 + // |- s3 + // |- s1 + // |- s3 + // |- s4 + + Schema nestedSchema = new Schema(); + nestedSchema.addColumn("s1", Type.INT8); + + nestedSchema.addColumn("s2", Type.INT8); + + Schema s5 = new Schema(); + s5.addColumn("s6", Type.INT8); + + Schema s7 = new Schema(); + s7.addColumn("s5", new TypeDesc(s5)); + + Schema s3 = new Schema(); + s3.addColumn("s4", Type.INT8); + s3.addColumn("s7", new TypeDesc(s7)); + s3.addColumn("s8", Type.INT8); + + nestedSchema.addColumn("s3", new TypeDesc(s3)); + nestedSchema.addColumn("s9", Type.INT8); + + assertSchemaEquality("nested_schema2", nestedSchema); + } + static IndexDesc desc1; static IndexDesc desc2; static IndexDesc desc3; http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java ---------------------------------------------------------------------- diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java index b7d9334..a3960e6 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java @@ -114,10 +114,7 @@ public class DescTableCommand extends TajoShellCommand { for(int i = 0; i < desc.getSchema().size(); i++) { Column col = desc.getSchema().getColumn(i); - sb.append(col.getSimpleName()).append("\t").append(col.getDataType().getType()); - if (col.getDataType().hasLength()) { - sb.append("(").append(col.getDataType().getLength()).append(")"); - } + sb.append(col.getSimpleName()).append("\t").append(col.getTypeDesc()); sb.append("\n"); } http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-common/src/main/proto/DataTypes.proto ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/proto/DataTypes.proto b/tajo-common/src/main/proto/DataTypes.proto index 04f1e12..fc5ac9a 100644 --- a/tajo-common/src/main/proto/DataTypes.proto +++ b/tajo-common/src/main/proto/DataTypes.proto @@ -101,5 +101,15 @@ enum Type { message DataType { required Type type = 1; optional int32 length = 2; + + /* Auxiliary information */ optional string code = 3; + + /** + * Nested fields. Since Protobuf does not support nested fields, + * the nesting is flattened to a single list by a depth-first traversal. + * The children count is used to construct the nested relationship. + * This field is not set when the element is a primitive type + */ + optional int32 num_nested_fields = 4; } http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml index 743180f..38bddec 100644 --- a/tajo-core/pom.xml +++ b/tajo-core/pom.xml @@ -492,6 +492,62 @@ </plugins> </build> </profile> + + <profile> <!-- This is for testing or debugging in IDE --> + <id>all-dependencies</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <dependencies> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <exclusions> + <exclusion> + <groupId>commons-el</groupId> + <artifactId>commons-el</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-runtime</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-compiler</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp-2.1-jetty</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey.jersey-test-framework</groupId> + <artifactId>jersey-test-framework-grizzly2</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-common</artifactId> + </dependency> + + </dependencies> + </profile> + <profile> <id>hcatalog-0.12.0</id> <activation> http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java index d6631f6..5819bd2 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java @@ -41,6 +41,27 @@ public class TestTajoDump extends QueryTestCaseBase { printWriter.close(); assertStrings(new String(bos.toByteArray())); bos.close(); + + executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName1\""); + } + } + + @Test + public void testDump2() throws Exception { + if (!testingCluster.isHCatalogStoreRunning()) { + executeString("CREATE TABLE \"" + getCurrentDatabase() + + "\".\"TableName2\" (\"Age\" int, \"Name\" Record (\"FirstName\" TEXT, lastname TEXT))"); + + UserRoleInfo userInfo = UserRoleInfo.getCurrentUser(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintWriter printWriter = new PrintWriter(bos); + TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter); + printWriter.flush(); + printWriter.close(); + assertStrings(new String(bos.toByteArray())); + bos.close(); + + executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName2\""); } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index d4a5a1f..aee5a02 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -205,17 +205,7 @@ public class TestTajoCli { assertEquals(databaseName, tajoCli.getContext().getCurrentDatabase()); } - @Test - public void testDescTable() throws Exception { - String tableName; - if (cluster.isHCatalogStoreRunning()) { - tableName = "TEST_DESC_TABLE".toLowerCase(); - } else { - tableName = "TEST_DESC_TABLE"; - } - - String sql = "create table \"" + tableName + "\" (col1 int4, col2 int4);"; - + private void verifyDescTable(String sql, String tableName, String resultFileName) throws Exception { setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); tajoCli.executeScript(sql); @@ -226,12 +216,38 @@ public class TestTajoCli { FileSystem fs = FileSystem.get(testBase.getTestingCluster().getConfiguration()); if (!cluster.isHCatalogStoreRunning()) { - assertOutputResult("testDescTable.result", consoleResult, new String[]{"${table.path}"}, + assertOutputResult(resultFileName, consoleResult, new String[]{"${table.path}"}, new String[]{fs.getUri() + "/tajo/warehouse/default/" + tableName}); } } @Test + public void testDescTable() throws Exception { + String tableName; + if (cluster.isHCatalogStoreRunning()) { + tableName = "TEST_DESC_TABLE".toLowerCase(); + } else { + tableName = "TEST_DESC_TABLE"; + } + + String sql = "create table \"" + tableName + "\" (col1 int4, col2 int4);"; + verifyDescTable(sql, tableName, "testDescTable.result"); + } + + @Test + public void testDescTableForNestedSchema() throws Exception { + String tableName; + if (cluster.isHCatalogStoreRunning()) { + tableName = "TEST_DESC_TABLE_NESTED".toLowerCase(); + } else { + tableName = "TEST_DESC_TABLE_NESTED"; + } + + String sql = "create table \"" + tableName + "\" (col1 int4, col2 int4, col3 record (col4 record (col5 text)));"; + verifyDescTable(sql, tableName, "testDescTableForNestedSchema.result"); + } + + @Test public void testSelectResultWithNullFalse() throws Exception { String sql = "select\n" + http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java index 21b3910..1fbe7c5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java @@ -600,4 +600,28 @@ public class TestCreateTable extends QueryTestCaseBase { */ } } + + @Test + public final void testNestedRecord1() throws Exception { + executeString("CREATE DATABASE D9;").close(); + + assertTableNotExists("d9.nested_table"); + executeQuery().close(); + assertTableExists("d9.nested_table"); + + executeString("DROP TABLE D9.nested_table"); + executeString("DROP DATABASE D9").close(); + } + + @Test + public final void testNestedRecord2() throws Exception { + executeString("CREATE DATABASE D9;").close(); + + assertTableNotExists("d9.nested_table2"); + executeQuery(); + assertTableExists("d9.nested_table2"); + + executeString("DROP TABLE D9.nested_table2"); + executeString("DROP DATABASE D9").close(); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/src/test/resources/queries/TestCreateTable/testNestedRecord1.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestCreateTable/testNestedRecord1.sql b/tajo-core/src/test/resources/queries/TestCreateTable/testNestedRecord1.sql new file mode 100644 index 0000000..d147361 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestCreateTable/testNestedRecord1.sql @@ -0,0 +1 @@ +CREATE TABLE D9.nested_table (f1 int, nested_field record (f2 int4, f3 int8), f3 text); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/src/test/resources/queries/TestCreateTable/testNestedRecord2.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestCreateTable/testNestedRecord2.sql b/tajo-core/src/test/resources/queries/TestCreateTable/testNestedRecord2.sql new file mode 100644 index 0000000..f794d21 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestCreateTable/testNestedRecord2.sql @@ -0,0 +1 @@ +CREATE TABLE D9.nested_table2 (f1 int, nf1 record (f1 int4, f3 double), nf2 record (f1 int4, nf1 record (f1 int4, f2 text), nf2 record (f1 int4, f2 text), f2 double), f2 text); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelect.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelect.result b/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelect.result index a6aa8f4..2dc746d 100644 --- a/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelect.result +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelect.result @@ -2,5 +2,5 @@ explain ------------------------------- SCAN(0) on default.lineitem => target list: default.lineitem.l_orderkey (INT4), default.lineitem.l_partkey (INT4) - => out schema: {(2) default.lineitem.l_orderkey (INT4),default.lineitem.l_partkey (INT4)} - => in schema: {(16) default.lineitem.l_orderkey (INT4),default.lineitem.l_partkey (INT4),default.lineitem.l_suppkey (INT4),default.lineitem.l_linenumber (INT4),default.lineitem.l_quantity (FLOAT8),default.lineitem.l_extendedprice (FLOAT8),default.lineitem.l_discount (FLOAT8),default.lineitem.l_tax (FLOAT8),default.lineitem.l_returnflag (TEXT),default.lineitem.l_linestatus (TEXT),default.lineitem.l_shipdate (TEXT),default.lineitem.l_commitdate (TEXT),default.lineitem.l_receiptdate (TEXT),default.lineitem.l_shipinstruct (TEXT),default.lineitem.l_shipmode (TEXT),default.lineitem.l_comment (TEXT)} \ No newline at end of file + => out schema: {(2) default.lineitem.l_orderkey (INT4), default.lineitem.l_partkey (INT4)} + => in schema: {(16) default.lineitem.l_orderkey (INT4), default.lineitem.l_partkey (INT4), default.lineitem.l_suppkey (INT4), default.lineitem.l_linenumber (INT4), default.lineitem.l_quantity (FLOAT8), default.lineitem.l_extendedprice (FLOAT8), default.lineitem.l_discount (FLOAT8), default.lineitem.l_tax (FLOAT8), default.lineitem.l_returnflag (TEXT), default.lineitem.l_linestatus (TEXT), default.lineitem.l_shipdate (TEXT), default.lineitem.l_commitdate (TEXT), default.lineitem.l_receiptdate (TEXT), default.lineitem.l_shipinstruct (TEXT), default.lineitem.l_shipmode (TEXT), default.lineitem.l_comment (TEXT)} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/src/test/resources/results/TestTajoCli/testDescTableForNestedSchema.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testDescTableForNestedSchema.result b/tajo-core/src/test/resources/results/TestTajoCli/testDescTableForNestedSchema.result new file mode 100644 index 0000000..83f360b --- /dev/null +++ b/tajo-core/src/test/resources/results/TestTajoCli/testDescTableForNestedSchema.result @@ -0,0 +1,29 @@ +OK + +table name: default.TEST_DESC_TABLE_NESTED +table path: ${table.path} +store type: CSV +number of rows: 0 +volume: 0 B +Options: + 'text.delimiter'='|' + +schema: +col1 INT4 +col2 INT4 +col3 RECORD (col4 RECORD (col5 TEXT)) + + + +table name: default.TEST_DESC_TABLE_NESTED +table path: ${table.path} +store type: CSV +number of rows: 0 +volume: 0 B +Options: + 'text.delimiter'='|' + +schema: +col1 INT4 +col2 INT4 +col3 RECORD (col4 RECORD (col5 TEXT)) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/src/test/resources/results/TestTajoDump/testDump2.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestTajoDump/testDump2.result b/tajo-core/src/test/resources/results/TestTajoDump/testDump2.result new file mode 100644 index 0000000..6c15e3e --- /dev/null +++ b/tajo-core/src/test/resources/results/TestTajoDump/testDump2.result @@ -0,0 +1,16 @@ +-- +-- Tajo database dump +-- + + +-- +-- Database name: "TestTajoDump" +-- + +CREATE DATABASE IF NOT EXISTS "TestTajoDump"; + +-- +-- Name: "TestTajoDump"."TableName2"; Type: TABLE; Storage: CSV +-- +CREATE TABLE "TestTajoDump"."TableName2" ("Age" INT4, "Name" RECORD ("FirstName" TEXT, lastname TEXT)) USING CSV WITH ('text.delimiter'='|'); + http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java index 235bebf..5166e80 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java @@ -793,10 +793,11 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva } return new ConstEval( - DatumFactory.cast(constEval.getValue(), LogicalPlanner.convertDataType(expr.getTarget()), tz)); + DatumFactory.cast(constEval.getValue(), + LogicalPlanner.convertDataType(expr.getTarget()).getDataType(), tz)); } else { - return new CastEval(ctx.queryContext, child, LogicalPlanner.convertDataType(expr.getTarget())); + return new CastEval(ctx.queryContext, child, LogicalPlanner.convertDataType(expr.getTarget()).getDataType()); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 14fea08..8395c3d 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -1860,7 +1860,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex * @param elements to be transformed * @return schema transformed from table definition elements */ - private Schema convertTableElementsSchema(ColumnDefinition[] elements) { + private static Schema convertTableElementsSchema(ColumnDefinition[] elements) { Schema schema = new Schema(); for (ColumnDefinition columnDefinition: elements) { @@ -1870,19 +1870,29 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex return schema; } - private Column convertColumn(ColumnDefinition columnDefinition) { + private static Column convertColumn(ColumnDefinition columnDefinition) { return new Column(columnDefinition.getColumnName(), convertDataType(columnDefinition)); } - public static TajoDataTypes.DataType convertDataType(DataTypeExpr dataType) { + public static TypeDesc convertDataType(DataTypeExpr dataType) { TajoDataTypes.Type type = TajoDataTypes.Type.valueOf(dataType.getTypeName()); TajoDataTypes.DataType.Builder builder = TajoDataTypes.DataType.newBuilder(); builder.setType(type); + if (dataType.hasLengthOrPrecision()) { builder.setLength(dataType.getLengthOrPrecision()); } - return builder.build(); + + TypeDesc typeDesc; + if (type == TajoDataTypes.Type.RECORD) { + Schema nestedRecordSchema = convertTableElementsSchema(dataType.getNestedRecordTypes()); + typeDesc = new TypeDesc(nestedRecordSchema); + } else { + typeDesc = new TypeDesc(builder.build()); + } + + return typeDesc; } http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java b/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java index 6222734..7c468bb 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java @@ -59,7 +59,7 @@ public class TypeDeterminant extends SimpleAlgebraVisitor<LogicalPlanner.PlanCon dataType = BOOL_TYPE; break; case Cast: - dataType = LogicalPlanner.convertDataType(((CastExpr)expr).getTarget()); + dataType = LogicalPlanner.convertDataType(((CastExpr)expr).getTarget()).getDataType(); break; default: dataType = visit(ctx, stack, expr.getChild()); @@ -270,7 +270,7 @@ public class TypeDeterminant extends SimpleAlgebraVisitor<LogicalPlanner.PlanCon @Override public DataType visitDataType(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, DataTypeExpr expr) throws PlanningException { - return LogicalPlanner.convertDataType(expr); + return LogicalPlanner.convertDataType(expr).getDataType(); } @Override
