TAJO-2108: Refactor Schema to be immutable. Closes #1000
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/4aef83a3 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/4aef83a3 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/4aef83a3 Branch: refs/heads/master Commit: 4aef83a39adf1055c212df5fac8b9451e24315b4 Parents: 71d2825 Author: Hyunsik Choi <[email protected]> Authored: Wed Apr 20 13:37:12 2016 -0700 Committer: Hyunsik Choi <[email protected]> Committed: Wed Apr 20 13:37:12 2016 -0700 ---------------------------------------------------------------------- CHANGES | 2 + .../org/apache/tajo/catalog/FieldConverter.java | 99 ++++++ .../apache/tajo/catalog/ListSchemaBuilder.java | 49 +++ .../java/org/apache/tajo/catalog/Schema.java | 14 - .../org/apache/tajo/catalog/SchemaBuilder.java | 190 +++++++++++ .../org/apache/tajo/catalog/SchemaFactory.java | 21 -- .../org/apache/tajo/catalog/SchemaLegacy.java | 30 +- .../org/apache/tajo/catalog/SchemaUtil.java | 35 +- .../apache/tajo/catalog/SetSchemaBuilder.java | 66 ++++ .../java/org/apache/tajo/catalog/TableDesc.java | 3 +- .../org/apache/tajo/catalog/TypeConverter.java | 101 ++++++ .../org/apache/tajo/catalog/TestIndexDesc.java | 4 +- .../org/apache/tajo/catalog/TestSchema.java | 172 +++++----- .../org/apache/tajo/catalog/TestTableDesc.java | 14 +- .../org/apache/tajo/catalog/TestTableMeta.java | 21 +- .../tajo/catalog/store/HiveCatalogStore.java | 30 +- .../catalog/store/TestHiveCatalogStore.java | 150 +++++---- .../tajo/catalog/store/AbstractDBStore.java | 9 +- .../apache/tajo/catalog/CatalogTestingUtil.java | 16 +- .../org/apache/tajo/catalog/TestCatalog.java | 169 +++++----- .../TestCatalogAgainstCaseSensitivity.java | 32 +- .../tajo/catalog/TestCatalogExceptions.java | 9 +- .../tajo/catalog/TestLinkedMetadataManager.java | 10 +- .../org/apache/tajo/client/TajoClientUtil.java | 6 +- .../apache/tajo/jdbc/TajoMetaDataResultSet.java | 9 +- .../org/apache/tajo/BackendTestingUtil.java | 9 +- .../java/org/apache/tajo/schema/Identifier.java | 16 + .../apache/tajo/schema/QualifiedIdentifier.java | 15 + .../java/org/apache/tajo/schema/Schema.java | 85 ++++- .../src/main/java/org/apache/tajo/type/Any.java | 33 ++ .../main/java/org/apache/tajo/type/Array.java | 5 + .../main/java/org/apache/tajo/type/Char.java | 5 + .../main/java/org/apache/tajo/type/Inet4.java | 28 ++ .../java/org/apache/tajo/type/Interval.java | 28 ++ .../main/java/org/apache/tajo/type/Null.java | 28 ++ .../java/org/apache/tajo/type/Protobuf.java | 42 +++ .../main/java/org/apache/tajo/type/Type.java | 25 ++ .../main/java/org/apache/tajo/type/Varchar.java | 5 + .../apache/tajo/cli/tools/TestDDLBuilder.java | 34 +- .../tajo/cli/tsql/TestTajoCliNegatives.java | 2 +- .../TestCatalogAdminClientExceptions.java | 9 +- .../engine/codegen/TestEvalCodeGenerator.java | 99 +++--- .../apache/tajo/engine/eval/TestEvalTree.java | 11 +- .../tajo/engine/eval/TestEvalTreeUtil.java | 11 +- .../apache/tajo/engine/eval/TestPredicates.java | 150 ++++----- .../tajo/engine/eval/TestSQLExpression.java | 121 ++----- .../engine/function/TestBuiltinFunctions.java | 241 +++++++------ .../function/TestConditionalExpressions.java | 52 +-- .../engine/function/TestDateTimeFunctions.java | 31 +- .../tajo/engine/function/TestMathFunctions.java | 260 +++++++------- .../function/TestPatternMatchingPredicates.java | 7 +- .../TestStringOperatorsAndFunctions.java | 220 ++++++------ .../function/TestUserDefinedFunctions.java | 7 +- .../engine/planner/TestJoinOrderAlgorithm.java | 42 ++- .../engine/planner/TestLogicalOptimizer.java | 31 +- .../tajo/engine/planner/TestLogicalPlanner.java | 69 ++-- .../tajo/engine/planner/TestPlannerUtil.java | 60 ++-- .../planner/TestUniformRangePartition.java | 203 ++++++----- .../planner/physical/TestExternalSortExec.java | 8 +- .../physical/TestFullOuterHashJoinExec.java | 40 ++- .../physical/TestFullOuterMergeJoinExec.java | 49 +-- .../planner/physical/TestHashAntiJoinExec.java | 24 +- .../planner/physical/TestHashJoinExec.java | 24 +- .../planner/physical/TestHashSemiJoinExec.java | 24 +- .../physical/TestLeftOuterHashJoinExec.java | 40 ++- .../planner/physical/TestMergeJoinExec.java | 24 +- .../planner/physical/TestPhysicalPlanner.java | 22 +- .../physical/TestProgressExternalSortExec.java | 9 +- .../engine/planner/physical/TestRadixSort.java | 6 +- .../physical/TestRightOuterHashJoinExec.java | 33 +- .../physical/TestRightOuterMergeJoinExec.java | 50 +-- .../engine/planner/physical/TestSortExec.java | 14 +- .../planner/physical/TestSortIntersectExec.java | 24 +- .../planner/physical/TestTupleSorter.java | 4 +- .../planner/physical/TestUnSafeTuple.java | 4 +- .../tajo/engine/query/TestGroupByQuery.java | 36 +- .../tajo/engine/query/TestHBaseTable.java | 79 +++-- .../apache/tajo/engine/query/TestJoinQuery.java | 42 ++- .../tajo/engine/query/TestNullValues.java | 75 +++-- .../tajo/engine/query/TestSelectQuery.java | 9 +- .../apache/tajo/engine/query/TestSortQuery.java | 58 ++-- .../tajo/engine/query/TestTablePartitions.java | 16 +- .../tajo/engine/query/TestWindowQuery.java | 39 ++- .../apache/tajo/engine/util/BenchmarkSort.java | 4 +- .../apache/tajo/engine/util/TestTupleUtil.java | 95 +++--- .../org/apache/tajo/storage/TestRowFile.java | 11 +- .../create_table_various_types.sql | 24 +- .../java/org/apache/tajo/benchmark/TPCH.java | 218 ++++++------ .../global/builder/DistinctGroupbyBuilder.java | 10 +- .../planner/physical/BSTIndexScanExec.java | 46 +-- .../planner/physical/ColPartitionStoreExec.java | 12 +- .../engine/planner/physical/SeqScanExec.java | 35 +- .../engine/planner/physical/WindowAggExec.java | 10 +- .../exec/ExplainPlanPreprocessorForTest.java | 7 +- .../apache/tajo/master/exec/QueryExecutor.java | 4 +- .../apache/tajo/querymaster/Repartitioner.java | 2 +- .../apache/tajo/jdbc/TajoDatabaseMetaData.java | 31 +- .../org/apache/tajo/jdbc/TestResultSet.java | 18 +- .../apache/tajo/jdbc/TestTajoJdbcNegative.java | 2 +- .../org/apache/tajo/plan/LogicalPlanner.java | 67 ++-- .../org/apache/tajo/plan/expr/EvalTreeUtil.java | 23 +- .../function/python/PythonScriptEngine.java | 24 +- .../tajo/plan/logical/CreateTableNode.java | 5 +- .../apache/tajo/plan/logical/InsertNode.java | 8 +- .../org/apache/tajo/plan/logical/ScanNode.java | 2 +- .../plan/rewrite/SelfDescSchemaBuildPhase.java | 26 +- .../tajo/plan/rewrite/rules/IndexScanInfo.java | 17 +- .../rewrite/rules/PartitionedTableRewriter.java | 7 +- .../org/apache/tajo/plan/util/PlannerUtil.java | 33 +- .../org/apache/tajo/plan/TestLogicalNode.java | 11 +- .../org/apache/tajo/storage/MergeScanner.java | 4 +- .../org/apache/tajo/storage/TupleRange.java | 19 +- .../org/apache/tajo/storage/TestLazyTuple.java | 67 ++-- .../tajo/storage/TestTupleComparator.java | 29 +- .../tajo/storage/hbase/TestColumnMapping.java | 13 +- .../tajo/storage/parquet/ParquetScanner.java | 5 +- .../storage/parquet/TajoSchemaConverter.java | 4 +- .../storage/thirdparty/orc/OrcRecordReader.java | 17 +- .../tajo/storage/TestCompressionStorages.java | 11 +- .../tajo/storage/TestDelimitedTextFile.java | 26 +- .../apache/tajo/storage/TestFileSystems.java | 11 +- .../apache/tajo/storage/TestFileTablespace.java | 38 ++- .../org/apache/tajo/storage/TestLineReader.java | 46 +-- .../apache/tajo/storage/TestMergeScanner.java | 20 +- .../org/apache/tajo/storage/TestStorages.java | 336 ++++++++++--------- .../apache/tajo/storage/index/TestBSTIndex.java | 90 ++--- .../index/TestSingleCSVFileBSTIndex.java | 27 +- .../apache/tajo/storage/json/TestJsonSerDe.java | 42 ++- .../tajo/storage/parquet/TestReadWrite.java | 43 ++- .../storage/parquet/TestSchemaConverter.java | 59 ++-- .../tajo/storage/raw/TestDirectRawFile.java | 31 +- .../storage/jdbc/JdbcMetadataProviderBase.java | 9 +- 132 files changed, 3333 insertions(+), 2364 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index ce8994d..244c261 100644 --- a/CHANGES +++ b/CHANGES @@ -10,6 +10,8 @@ Release 0.12.0 - unreleased IMPROVEMENT + TAJO-2108: Refactor Schema to be immutable. (hyunsik) + TAJO-2112: Improve disk load, when queries run simultaneously. (jinho) TAJO-2104: Implement Identifier which supports quotation information. http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java new file mode 100644 index 0000000..616b44e --- /dev/null +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog; + +import com.google.common.base.Function; +import com.google.common.collect.Collections2; +import com.google.common.collect.ImmutableList; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.exception.NotImplementedException; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.schema.Identifier; +import org.apache.tajo.schema.IdentifierPolicy; +import org.apache.tajo.schema.QualifiedIdentifier; +import org.apache.tajo.schema.Schema; +import org.apache.tajo.schema.Schema.NamedPrimitiveType; +import org.apache.tajo.schema.Schema.NamedStructType; +import org.apache.tajo.type.*; + +import javax.annotation.Nullable; +import java.util.Collection; + +public class FieldConverter { + + public static QualifiedIdentifier toQualifiedIdentifier(String name) { + final Collection<String> elems = ImmutableList.copyOf(name.split("\\.")); + final Collection<Identifier> identifiers = Collections2.transform(elems, new Function<String, Identifier>() { + @Override + public Identifier apply(@Nullable String input) { + boolean needQuote = CatalogUtil.isShouldBeQuoted(input); + return Identifier._(input, needQuote); + } + }); + return QualifiedIdentifier.$(identifiers); + } + + public static TypeDesc convert(Schema.NamedType src) { + if (src instanceof NamedStructType) { + NamedStructType structType = (NamedStructType) src; + + ImmutableList.Builder<Column> fields = ImmutableList.builder(); + for (Schema.NamedType t: structType.fields()) { + fields.add(new Column(t.name().raw(IdentifierPolicy.DefaultPolicy()), convert(t))); + } + + return new TypeDesc(SchemaBuilder.builder().addAll(fields.build()).build()); + } else { + final NamedPrimitiveType namedType = (NamedPrimitiveType) src; + final Type type = namedType.type(); + if (type instanceof Char) { + Char charType = (Char) type; + return new TypeDesc(CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.CHAR, charType.length())); + } else if (type instanceof Varchar) { + Varchar varcharType = (Varchar) type; + return new TypeDesc(CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.VARCHAR, varcharType.length())); + } else if (type instanceof Numeric) { + Numeric numericType = (Numeric) type; + return new TypeDesc(CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.NUMERIC, numericType.precision())); + } else if (type instanceof Protobuf) { + Protobuf protobuf = (Protobuf) type; + return new TypeDesc(CatalogUtil.newDataType(TajoDataTypes.Type.PROTOBUF, protobuf.getMessageName())); + } else { + return new TypeDesc(TypeConverter.convert(namedType.type())); + } + } + } + + public static Schema.NamedType convert(Column column) { + if (column.getTypeDesc().getDataType().getType() == TajoDataTypes.Type.RECORD) { + + if (column.getTypeDesc().getNestedSchema() == null) { + throw new TajoRuntimeException(new NotImplementedException("record type projection")); + } + + return new NamedStructType(toQualifiedIdentifier(column.getQualifiedName()), + TypeConverter.convert(column.getTypeDesc())); + + } else { + return new NamedPrimitiveType(toQualifiedIdentifier(column.getQualifiedName()), + TypeConverter.convert(column.getDataType()) + ); + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/ListSchemaBuilder.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/ListSchemaBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/ListSchemaBuilder.java new file mode 100644 index 0000000..e0c9d9e --- /dev/null +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/ListSchemaBuilder.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import org.apache.tajo.schema.Schema.NamedType; + +import java.util.Iterator; + +public class ListSchemaBuilder implements SchemaBuilder.SchemaCollector { + private final ImmutableList.Builder<NamedType> fields = new ImmutableList.Builder(); + + @Override + public void add(NamedType namedType) { + fields.add(namedType); + } + + @Override + public void addAll(Iterator<NamedType> fields) { + this.fields.addAll(fields); + } + + @Override + public void addAll(Iterable<NamedType> fields) { + this.fields.addAll(fields); + } + + @Override + public ImmutableCollection<NamedType> build() { + return fields.build(); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java index 1b4c1eb..ec6af2c 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java @@ -20,8 +20,6 @@ package org.apache.tajo.catalog; import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto; import org.apache.tajo.common.ProtoObject; -import org.apache.tajo.common.TajoDataTypes.DataType; -import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.json.GsonObject; import java.util.Collection; @@ -90,18 +88,6 @@ public interface Schema extends ProtoObject<SchemaProto>, Cloneable, GsonObject */ boolean containsAny(Collection<Column> columns); - Schema addColumn(String name, TypeDesc typeDesc); - - Schema addColumn(String name, Type type); - - Schema addColumn(String name, Type type, int length); - - Schema addColumn(String name, DataType dataType); - - void addColumn(Column column); - - void addColumns(Schema schema); - @Override boolean equals(Object o); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java new file mode 100644 index 0000000..35e38b5 --- /dev/null +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java @@ -0,0 +1,190 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.schema.QualifiedIdentifier; +import org.apache.tajo.schema.Schema.NamedPrimitiveType; +import org.apache.tajo.schema.Schema.NamedStructType; +import org.apache.tajo.schema.Schema.NamedType; +import org.apache.tajo.type.Type; + +import javax.annotation.Nullable; +import java.util.Collection; +import java.util.Iterator; + +import static org.apache.tajo.catalog.FieldConverter.toQualifiedIdentifier; +import static org.apache.tajo.schema.IdentifierPolicy.DefaultPolicy; + +/** + * Builder for Schema + */ +public class SchemaBuilder { + private final SchemaCollector fields; + + public interface SchemaCollector { + void add(NamedType field); + void addAll(Iterator<NamedType> fields); + void addAll(Iterable<NamedType> fields); + ImmutableCollection<NamedType> build(); + } + + public static SchemaLegacy empty() { + return builder().build(); + } + + public static SchemaBuilder builder() { + return new SchemaBuilder(new ListSchemaBuilder()); + } + + public static SchemaBuilder uniqueNameBuilder() { + return new SchemaBuilder(new SetSchemaBuilder()); + } + + SchemaBuilder(SchemaCollector collector) { + this.fields = collector; + } + + public SchemaBuilder add(NamedType namedType) { + fields.add(namedType); + return this; + } + + public SchemaBuilder add(QualifiedIdentifier id, Type type) { + add(new NamedPrimitiveType(id, type)); + return this; + } + + public SchemaBuilder addStruct(QualifiedIdentifier id, Collection<NamedType> fields) { + add(new NamedStructType(id, fields)); + return this; + } + + @Deprecated + public SchemaBuilder add(String name, TypeDesc legacyType) { + if (legacyType.getDataType().getType() == TajoDataTypes.Type.RECORD) { + addStruct(toQualifiedIdentifier(name), TypeConverter.convert(legacyType)); + } else { + add(toQualifiedIdentifier(name), TypeConverter.convert(legacyType.getDataType())); + } + return this; + } + + @Deprecated + public SchemaBuilder add(String name, TajoDataTypes.DataType dataType) { + add(name, new TypeDesc(dataType)); + return this; + } + + @Deprecated + public SchemaBuilder add(String name, TajoDataTypes.Type baseType) { + add(name, new TypeDesc(CatalogUtil.newSimpleDataType(baseType))); + return this; + } + + @Deprecated + public SchemaBuilder add(Column column) { + add(FieldConverter.convert(column)); + return this; + } + + @Deprecated + public SchemaBuilder addAll(Iterable<Column> columns) { + return addAll2(columns, new Function<Column, NamedType>() { + @Override + public NamedType apply(@Nullable Column input) { + return FieldConverter.convert(input); + } + }); + } + + @Deprecated + public SchemaBuilder addAll(Column [] columns) { + return addAll2(columns, new Function<Column, NamedType>() { + @Override + public NamedType apply(@Nullable Column input) { + return FieldConverter.convert(input); + } + }); + } + + @Deprecated + public <T> SchemaBuilder addAll(T [] fields, Function<T, Column> fn) { + for (T t : fields) { + add(fn.apply(t)); + } + return this; + } + + @Deprecated + public <T> SchemaBuilder addAll(Iterable<T> fields, Function<T, Column> fn) { + for (T t : fields) { + add(fn.apply(t)); + } + return this; + } + + @Deprecated + public <T> SchemaBuilder addAll(Iterator<T> fields, Function<T, Column> fn) { + while(fields.hasNext()) { + T t = fields.next(); + add(fn.apply(t)); + } + return this; + } + + public <T> SchemaBuilder addAll2(T [] fields, Function<T, NamedType> fn) { + for (T t : fields) { + add(fn.apply(t)); + } + return this; + } + + public <T> SchemaBuilder addAll2(Iterable<T> fields, Function<T, NamedType> fn) { + for (T t : fields) { + add(fn.apply(t)); + } + return this; + } + + public <T> SchemaBuilder addAll2(Iterator<T> fields, Function<T, NamedType> fn) { + while(fields.hasNext()) { + T t = fields.next(); + add(fn.apply(t)); + } + return this; + } + + @Deprecated + public SchemaLegacy build() { + ImmutableList.Builder<Column> columns = new ImmutableList.Builder(); + for (NamedType namedType : fields.build()) { + columns.add(new Column(namedType.name().raw(DefaultPolicy()), FieldConverter.convert(namedType))); + } + + return new SchemaLegacy(columns.build()); + } + + public org.apache.tajo.schema.Schema buildV2() { + return new org.apache.tajo.schema.Schema(fields.build()); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaFactory.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaFactory.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaFactory.java index fb6910b..d2a5a06 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaFactory.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaFactory.java @@ -19,30 +19,9 @@ package org.apache.tajo.catalog; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.exception.TajoInternalError; public class SchemaFactory { - public static Schema newV1() { - return new SchemaLegacy(); - } - public static Schema newV1(CatalogProtos.SchemaProto proto) { return new SchemaLegacy(proto); } - - public static Schema newV1(Schema schema) { - try { - return (Schema) schema.clone(); - } catch (CloneNotSupportedException e) { - throw new TajoInternalError(e); - } - } - - public static Schema newV1(Column [] columns) { - return new SchemaLegacy(columns); - } - - public static Schema newV1(Iterable<Column> columns) { - return new SchemaLegacy(columns); - } } http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java index f23d519..2a16fa6 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java @@ -424,8 +424,7 @@ public class SchemaLegacy implements Schema, ProtoObject<SchemaProto>, Cloneable return false; } - @Override - public synchronized SchemaLegacy addColumn(String name, TypeDesc typeDesc) { + private SchemaLegacy addColumn(String name, TypeDesc typeDesc) { String normalized = name; if(fieldsByQualifiedName.containsKey(normalized)) { throw new TajoRuntimeException(new DuplicateColumnException(normalized)); @@ -441,36 +440,11 @@ public class SchemaLegacy implements Schema, ProtoObject<SchemaProto>, Cloneable return this; } - @Override - public synchronized SchemaLegacy addColumn(String name, Type type) { - return addColumn(name, CatalogUtil.newSimpleDataType(type)); - } - - @Override - public synchronized SchemaLegacy addColumn(String name, Type type, int length) { - return addColumn(name, CatalogUtil.newDataTypeWithLen(type, length)); - } - - @Override - public synchronized SchemaLegacy addColumn(String name, DataType dataType) { - addColumn(name, new TypeDesc(dataType)); - - return this; - } - - @Override - public synchronized void addColumn(Column column) { + private synchronized void addColumn(Column column) { addColumn(column.getQualifiedName(), column.typeDesc); } @Override - public synchronized void addColumns(Schema schema) { - for(Column column : schema.getRootColumns()) { - addColumn(column); - } - } - - @Override public int hashCode() { return Objects.hashCode(fields, fieldsByQualifiedName, fieldsByName); } http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java index 0c62ae5..c0b60a3 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java @@ -23,9 +23,7 @@ import com.google.common.collect.Lists; import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UnsupportedDataTypeException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; +import java.util.*; import static org.apache.tajo.common.TajoDataTypes.DataType; import static org.apache.tajo.common.TajoDataTypes.Type; @@ -42,17 +40,19 @@ public class SchemaUtil { // The essential solution would be https://issues.apache.org/jira/browse/TAJO-895. static int tmpColumnSeq = 0; public static Schema merge(Schema left, Schema right) { - Schema merged = SchemaFactory.newV1(); + SchemaBuilder merged = SchemaBuilder.builder(); + Set<String> nameSet = new HashSet<>(); + for(Column col : left.getRootColumns()) { - if (!merged.containsByQualifiedName(col.getQualifiedName())) { - merged.addColumn(col); - } + merged.add(col); + nameSet.add(col.getQualifiedName()); } for(Column col : right.getRootColumns()) { - if (merged.containsByQualifiedName(col.getQualifiedName())) { - merged.addColumn("?fake" + (tmpColumnSeq++), col.getDataType()); + if (nameSet.contains(col.getQualifiedName())) { + merged.add("?fake" + (tmpColumnSeq++), col.getDataType()); } else { - merged.addColumn(col); + merged.add(col); + nameSet.add(col.getQualifiedName()); } } @@ -60,25 +60,28 @@ public class SchemaUtil { if (tmpColumnSeq < 0) { tmpColumnSeq = 0; } - return merged; + return merged.build(); } /** * Get common columns to be used as join keys of natural joins. */ public static Schema getNaturalJoinColumns(Schema left, Schema right) { - Schema common = SchemaFactory.newV1(); + + SchemaBuilder common = SchemaBuilder.builder(); + Set<String> commonNames = new HashSet<>(); for (Column outer : left.getRootColumns()) { - if (!common.containsByName(outer.getSimpleName()) && right.containsByName(outer.getSimpleName())) { - common.addColumn(new Column(outer.getSimpleName(), outer.getDataType())); + if (!commonNames.contains(outer.getSimpleName()) && right.containsByName(outer.getSimpleName())) { + common.add(new Column(outer.getSimpleName(), outer.getDataType())); + commonNames.add(outer.getSimpleName()); } } - return common; + return common.build(); } public static Schema getQualifiedLogicalSchema(TableDesc tableDesc, String tableName) { - Schema logicalSchema = SchemaFactory.newV1(tableDesc.getLogicalSchema()); + Schema logicalSchema = SchemaBuilder.builder().addAll(tableDesc.getLogicalSchema().getRootColumns()).build(); if (tableName != null) { logicalSchema.setQualifier(tableName); } http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SetSchemaBuilder.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SetSchemaBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SetSchemaBuilder.java new file mode 100644 index 0000000..b66aac2 --- /dev/null +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SetSchemaBuilder.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import org.apache.commons.collections.collection.UnmodifiableCollection; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.schema.QualifiedIdentifier; +import org.apache.tajo.schema.Schema.NamedPrimitiveType; +import org.apache.tajo.schema.Schema.NamedStructType; +import org.apache.tajo.schema.Schema.NamedType; +import org.apache.tajo.type.Type; + +import java.util.*; + +import static org.apache.tajo.catalog.FieldConverter.toQualifiedIdentifier; +import static org.apache.tajo.schema.IdentifierPolicy.DefaultPolicy; + +public class SetSchemaBuilder implements SchemaBuilder.SchemaCollector { + private final Set<QualifiedIdentifier> nameSet = new HashSet<>(); + private final ImmutableList.Builder<NamedType> fields = new ImmutableList.Builder(); + + @Override + public void add(NamedType namedType) { + if (!nameSet.contains(namedType.name())) { + fields.add(namedType); + nameSet.add(namedType.name()); + } + } + + @Override + public void addAll(Iterator<NamedType> fields) { + while (fields.hasNext()) { + add(fields.next()); + } + } + + @Override + public void addAll(Iterable<NamedType> fields) { + for (NamedType n : fields) { + add(n); + } + } + + @Override + public ImmutableCollection<NamedType> build() { + return fields.build(); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java index 392a83d..8122bd5 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java @@ -122,8 +122,7 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone public Schema getLogicalSchema() { if (hasPartition()) { - Schema logicalSchema = SchemaFactory.newV1(schema); - logicalSchema.addColumns(getPartitionMethod().getExpressionSchema()); + Schema logicalSchema = SchemaUtil.merge(schema, getPartitionMethod().getExpressionSchema()); logicalSchema.setQualifier(tableName); return logicalSchema; } else { http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java new file mode 100644 index 0000000..f82cc03 --- /dev/null +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog; + +import com.google.common.collect.ImmutableList; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.schema.Schema; +import org.apache.tajo.type.Protobuf; +import org.apache.tajo.type.Type; + +import java.util.Collection; + +import static org.apache.tajo.type.Type.*; + +public class TypeConverter { + + public static Collection<Schema.NamedType> convert(TypeDesc type) { + ImmutableList.Builder<Schema.NamedType> fields = ImmutableList.builder(); + for (Column c : type.getNestedSchema().getRootColumns()) { + fields.add(FieldConverter.convert(c)); + } + return fields.build(); + } + + public static Type convert(TajoDataTypes.Type legacyBaseType) { + switch (legacyBaseType) { + case BOOLEAN: + return Bool(); + case INT1: + case INT2: + return Int2(); + case INT4: + return Int4(); + case INT8: + return Int8(); + case FLOAT4: + return Float4(); + case FLOAT8: + return Float8(); + case DATE: + return Date(); + case TIME: + return Time(); + case TIMESTAMP: + return Timestamp(); + case INTERVAL: + return Interval(); + case TEXT: + return Text(); + case BLOB: + return Blob(); + case INET4: + return Inet4(); + case NULL_TYPE: + return Null(); + case ANY: + return Any(); + default: + throw new TajoRuntimeException(new UnsupportedException(legacyBaseType.name())); + } + } + + public static Type convert(TajoDataTypes.DataType legacyType) { + switch (legacyType.getType()) { + case NCHAR: + case CHAR: + return Char(legacyType.getLength()); + case NVARCHAR: + case VARCHAR: + return Varchar(legacyType.getLength()); + case NUMERIC: + return Numeric(legacyType.getLength()); + case PROTOBUF: + return new Protobuf(legacyType.getCode()); + default: + return convert(legacyType.getType()); + } + } + + public static TajoDataTypes.DataType convert(Type type) { + return CatalogUtil.newSimpleDataType(type.baseType()); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java index 0b7516e..52c68f0 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java @@ -40,8 +40,8 @@ public class TestIndexDesc { @BeforeClass public static void setUp() throws Exception { - relationSchema = SchemaFactory.newV1(new Column[]{new Column("id", Type.INT4), - new Column("score", Type.FLOAT8), new Column("name", Type.TEXT)}); + relationSchema = SchemaBuilder.builder().addAll(new Column[]{new Column("id", Type.INT4), + new Column("score", Type.FLOAT8), new Column("name", Type.TEXT)}).build(); SortSpec[] colSpecs1 = new SortSpec[1]; colSpecs1[0] = new SortSpec(new Column("id", Type.INT4), true, true); desc1 = new IndexDesc(DEFAULT_DATABASE_NAME, "indexed", http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java index 6235945..a474730 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java @@ -40,18 +40,19 @@ public class TestSchema { static { // simple nested schema - nestedSchema1 = SchemaFactory.newV1(); - nestedSchema1.addColumn("s1", Type.INT8); + SchemaBuilder builder1 = SchemaBuilder.builder(); + builder1.add(new Column("s1", Type.INT8)); - Schema nestedRecordSchema = SchemaFactory.newV1(); - nestedRecordSchema.addColumn("s2", Type.FLOAT4); - nestedRecordSchema.addColumn("s3", Type.TEXT); + Schema nestedRecordSchema = SchemaBuilder.builder() + .add("s2", Type.FLOAT4) + .add("s3", Type.TEXT) + .build(); Column nestedField = new Column("s4", new TypeDesc(nestedRecordSchema)); - nestedSchema1.addColumn(nestedField); - - nestedSchema1.addColumn("s5", Type.FLOAT8); + builder1.add(nestedField); + builder1.add(new Column("s5", Type.FLOAT8)); + nestedSchema1 = builder1.build(); // two level nested schema // // s1 @@ -63,25 +64,27 @@ public class TestSchema { // |- s8 // |- s6 // |- s7 - nestedSchema2 = SchemaFactory.newV1(); - nestedSchema2.addColumn("s1", Type.INT8); + SchemaBuilder builder2 = SchemaBuilder.builder(); + builder2.add(new Column("s1", Type.INT8)); - Schema nestedRecordSchema1 = SchemaFactory.newV1(); - nestedRecordSchema1.addColumn("s2", Type.FLOAT4); - nestedRecordSchema1.addColumn("s3", Type.TEXT); + Schema nestedRecordSchema1 = SchemaBuilder.builder() + .add("s2", Type.FLOAT4) + .add("s3", Type.TEXT) + .build(); Column nestedField1 = new Column("s4", new TypeDesc(nestedRecordSchema1)); - nestedSchema2.addColumn(nestedField1); + builder2.add(nestedField1); - nestedSchema2.addColumn("s5", Type.FLOAT8); + builder2.add(new Column("s5", Type.FLOAT8)); - Schema nestedRecordSchema2 = SchemaFactory.newV1(); - nestedRecordSchema2.addColumn("s6", Type.FLOAT4); - nestedRecordSchema2.addColumn("s7", Type.TEXT); + Schema nestedRecordSchema2 = SchemaBuilder.builder() + .add("s6", Type.FLOAT4) + .add("s7", Type.TEXT) + .build(); Column nestedField2 = new Column("s8", new TypeDesc(nestedRecordSchema2)); - nestedSchema2.addColumn(nestedField2); - + builder2.add(nestedField2); + nestedSchema2 = builder2.build(); // three level nested schema // @@ -95,41 +98,42 @@ public class TestSchema { // |- s8 // |- s9 - nestedSchema3 = SchemaFactory.newV1(); - nestedSchema3.addColumn("s1", Type.INT8); + SchemaBuilder builder3 = SchemaBuilder.builder(); - nestedSchema3.addColumn("s2", Type.INT8); + builder3.add("s1", Type.INT8); + builder3.add("s2", Type.INT8); - Schema s5 = SchemaFactory.newV1(); - s5.addColumn("s6", Type.INT8); + SchemaBuilder s5 = SchemaBuilder.builder(); + s5.add("s6", Type.INT8); - Schema s7 = SchemaFactory.newV1(); - s7.addColumn("s5", new TypeDesc(s5)); + SchemaBuilder s7 = SchemaBuilder.builder(); + s7.add("s5", new TypeDesc(s5.build())); - Schema s3 = SchemaFactory.newV1(); - s3.addColumn("s4", Type.INT8); - s3.addColumn("s7", new TypeDesc(s7)); - s3.addColumn("s8", Type.INT8); + SchemaBuilder s3 = SchemaBuilder.builder(); + s3.add("s4", Type.INT8); + s3.add("s7", new TypeDesc(s7.build())); + s3.add("s8", Type.INT8); - nestedSchema3.addColumn("s3", new TypeDesc(s3)); - nestedSchema3.addColumn("s9", Type.INT8); + builder3.add(new Column("s3", new TypeDesc(s3.build()))); + builder3.add(new Column("s9", Type.INT8)); + nestedSchema3 = builder3.build(); } @Before public void setUp() throws Exception { - schema = SchemaFactory.newV1(); + SchemaBuilder schemaBld = SchemaBuilder.builder(); col1 = new Column("name", Type.TEXT); - schema.addColumn(col1); + schemaBld.add(col1); col2 = new Column("age", Type.INT4); - schema.addColumn(col2); + schemaBld.add(col2); col3 = new Column("addr", Type.TEXT); - schema.addColumn(col3); + schemaBld.add(col3); + schema = schemaBld.build(); } @Test public final void testSchemaSchema() { - Schema schema2 = SchemaFactory.newV1(schema); - + Schema schema2 = SchemaBuilder.builder().addAll(schema.getRootColumns()).build(); assertEquals(schema, schema2); } @@ -149,20 +153,21 @@ public class TestSchema { @Test public final void testAddField() { - Schema schema = SchemaFactory.newV1(); + Schema schema = SchemaBuilder.builder().build(); assertFalse(schema.containsByQualifiedName("studentId")); - schema.addColumn("studentId", Type.INT4); - assertTrue(schema.containsByQualifiedName("studentId")); + Schema schema2 = SchemaBuilder.builder().addAll(schema.getRootColumns()).add("studentId", Type.INT4).build(); + assertTrue(schema2.containsByQualifiedName("studentId")); } @Test public final void testEqualsObject() { - Schema schema2 = SchemaFactory.newV1(); - schema2.addColumn("name", Type.TEXT); - schema2.addColumn("age", Type.INT4); - schema2.addColumn("addr", Type.TEXT); - - assertEquals(schema, schema2); + Schema schema2 = SchemaBuilder.builder() + .add("name", Type.TEXT) + .add("age", Type.INT4) + .add("addr", Type.TEXT) + .build(); + + assertEquals(schema, schema2); } @Test @@ -176,11 +181,12 @@ public class TestSchema { @Test public final void testClone() throws CloneNotSupportedException { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("abc", Type.FLOAT8); - schema.addColumn("bbc", Type.FLOAT8); - - Schema schema2 = SchemaFactory.newV1(schema.getProto()); + Schema schema = SchemaBuilder.builder() + .add("abc", Type.FLOAT8) + .add("bbc", Type.FLOAT8) + .build(); + + Schema schema2 = SchemaFactory.newV1(schema.getProto()); assertEquals(schema.getProto(), schema2.getProto()); assertEquals(schema.getColumn(0), schema2.getColumn(0)); assertEquals(schema.size(), schema2.size()); @@ -193,11 +199,12 @@ public class TestSchema { @Test(expected = TajoRuntimeException.class) public final void testAddExistColumn() { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("abc", Type.FLOAT8); - schema.addColumn("bbc", Type.FLOAT8); - schema.addColumn("abc", Type.INT4); - } + SchemaBuilder.builder() + .add("abc", Type.FLOAT8) + .add("bbc", Type.FLOAT8) + .add("abc", Type.INT4) + .build(); + } @Test public final void testJson() { @@ -225,9 +232,10 @@ public class TestSchema { assertEquals(column, schema2.getColumn("age")); assertEquals(column, schema2.getColumn("test1.age")); - Schema schema3 = SchemaFactory.newV1(); - schema3.addColumn("tb1.col1", Type.INT4); - schema3.addColumn("col2", Type.INT4); + Schema schema3 = SchemaBuilder.builder() + .add("tb1.col1", Type.INT4) + .add("col2", Type.INT4) + .build(); assertEquals("tb1", schema3.getColumn(0).getQualifier()); assertEquals("tb1.col1", schema3.getColumn(0).getQualifiedName()); assertEquals("col1", schema3.getColumn(0).getSimpleName()); @@ -267,26 +275,28 @@ public class TestSchema { @Test public void testNestedRecord4() { - Schema root = SchemaFactory.newV1(); - - Schema nf2DotNf1 = SchemaFactory.newV1(); - nf2DotNf1.addColumn("f1", Type.INT8); - nf2DotNf1.addColumn("f2", Type.INT8); - - Schema nf2DotNf2 = SchemaFactory.newV1(); - nf2DotNf2.addColumn("f1", Type.INT8); - nf2DotNf2.addColumn("f2", Type.INT8); - - Schema nf2 = SchemaFactory.newV1(); - nf2.addColumn("f1", Type.INT8); - nf2.addColumn("nf1", new TypeDesc(nf2DotNf1)); - nf2.addColumn("nf2", new TypeDesc(nf2DotNf2)); - nf2.addColumn("f2", Type.INT8); - - root.addColumn("f1", Type.INT8); - root.addColumn("nf1", Type.INT8); - root.addColumn("nf2", new TypeDesc(nf2)); - root.addColumn("f2", Type.INT8); + + Schema nf2DotNf1 = SchemaBuilder.builder() + .add("f1", Type.INT8) + .add("f2", Type.INT8) + .build(); + + Schema nf2DotNf2 = SchemaBuilder.builder() + .add("f1", Type.INT8) + .add("f2", Type.INT8) + .build(); + + Schema nf2 = SchemaBuilder.builder() + .add("f1", Type.INT8) + .add("nf1", new TypeDesc(nf2DotNf1)) + .add("nf2", new TypeDesc(nf2DotNf2)) + .add("f2", Type.INT8).build(); + + Schema root = SchemaBuilder.builder() + .add("f1", Type.INT8) + .add("nf1", Type.INT8) + .add("nf2", new TypeDesc(nf2)) + .add("f2", Type.INT8).build(); verifySchema(root); } http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java index f334738..085975b 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java @@ -41,9 +41,10 @@ public class TestTableDesc { @Before public void setup() throws IOException { - schema = SchemaFactory.newV1(); - schema.addColumn("name", Type.BLOB); - schema.addColumn("addr", Type.TEXT); + schema = SchemaBuilder.builder() + .add("name", Type.BLOB) + .add("addr", Type.TEXT) + .build(); info = CatalogUtil.newTableMeta("TEXT"); path = new Path(CommonTestingUtil.getTestDir(), "table1"); desc = new TableDesc("table1", schema, info, path.toUri()); @@ -67,9 +68,10 @@ public class TestTableDesc { @Test public void test() throws CloneNotSupportedException, IOException { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("name", Type.BLOB); - schema.addColumn("addr", Type.TEXT); + Schema schema = SchemaBuilder.builder() + .add("name", Type.BLOB) + .add("addr", Type.TEXT) + .build(); TableMeta info = CatalogUtil.newTableMeta("TEXT"); testClone(info); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java index 2e4c6a9..c6f4d1a 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java @@ -21,7 +21,6 @@ package org.apache.tajo.catalog; import org.apache.tajo.BuiltinStorages; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos.TableProto; -import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.junit.Before; import org.junit.Test; @@ -37,10 +36,7 @@ public class TestTableMeta { } @Test - public void testTableMetaTableProto() { - Schema schema1 = SchemaFactory.newV1(); - schema1.addColumn("name", Type.BLOB); - schema1.addColumn("addr", Type.TEXT); + public void testTableMetaTableProto() { TableMeta meta1 = CatalogUtil.newTableMeta("TEXT"); TableMeta meta2 = new TableMeta(meta1.getProto()); @@ -49,9 +45,6 @@ public class TestTableMeta { @Test public final void testClone() throws CloneNotSupportedException { - Schema schema1 = SchemaFactory.newV1(); - schema1.addColumn("name", Type.BLOB); - schema1.addColumn("addr", Type.TEXT); TableMeta meta1 = CatalogUtil.newTableMeta("TEXT"); TableMeta meta2 = (TableMeta) meta1.clone(); @@ -61,13 +54,8 @@ public class TestTableMeta { @Test public void testSchema() throws CloneNotSupportedException { - Schema schema1 = SchemaFactory.newV1(); - schema1.addColumn("name", Type.BLOB); - schema1.addColumn("addr", Type.TEXT); TableMeta meta1 = CatalogUtil.newTableMeta("TEXT"); - TableMeta meta2 = (TableMeta) meta1.clone(); - assertEquals(meta1, meta2); } @@ -77,13 +65,8 @@ public class TestTableMeta { } @Test - public void testEqualsObject() { - Schema schema2 = SchemaFactory.newV1(); - schema2.addColumn("name", Type.BLOB); - schema2.addColumn("addr", Type.TEXT); + public void testEqualsObject() { TableMeta meta2 = CatalogUtil.newTableMeta("TEXT"); - - assertTrue(meta.equals(meta2)); assertNotSame(meta, meta2); } http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 954817c..6aa0585 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -46,6 +46,7 @@ import org.apache.tajo.algebra.Expr; import org.apache.tajo.algebra.IsNullPredicate; import org.apache.tajo.algebra.JsonHelper; import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -134,7 +135,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { org.apache.hadoop.hive.ql.metadata.Table table = null; Path path = null; String dataFormat = null; - org.apache.tajo.catalog.Schema schema = null; + Schema schema = null; KeyValueSet options = null; TableStats stats = null; PartitionMethodDesc partitions = null; @@ -148,7 +149,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { path = table.getPath(); // convert HiveCatalogStore field schema into tajo field schema. - schema = SchemaFactory.newV1(); + SchemaBuilder schemaBuilder = SchemaBuilder.builder(); List<FieldSchema> fieldSchemaList = table.getCols(); boolean isPartitionKey; @@ -167,9 +168,10 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + CatalogConstants.IDENTIFIER_DELIMITER + eachField.getName(); TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(eachField.getType()); - schema.addColumn(fieldName, dataType); + schemaBuilder.add(fieldName, dataType); } } + schema = schemaBuilder.build(); // validate field schema. HiveCatalogUtil.validateSchema(table); @@ -238,7 +240,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { List<FieldSchema> partitionKeys = table.getPartitionKeys(); if (null != partitionKeys) { - org.apache.tajo.catalog.Schema expressionSchema = SchemaFactory.newV1(); + SchemaBuilder expressionSchema = SchemaBuilder.builder(); StringBuilder sb = new StringBuilder(); if (partitionKeys.size() > 0) { for (int i = 0; i < partitionKeys.size(); i++) { @@ -246,7 +248,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(fieldSchema.getType()); String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + CatalogConstants.IDENTIFIER_DELIMITER + fieldSchema.getName(); - expressionSchema.addColumn(new Column(fieldName, dataType)); + expressionSchema.add(new Column(fieldName, dataType)); if (i > 0) { sb.append(","); } @@ -257,7 +259,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { tableName, PartitionType.COLUMN, sb.toString(), - expressionSchema); + expressionSchema.build()); } } } catch (Throwable t) { @@ -278,16 +280,6 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { return tableDesc.getProto(); } - - private TajoDataTypes.Type getDataType(final String typeStr) { - try { - return Enum.valueOf(TajoDataTypes.Type.class, typeStr); - } catch (IllegalArgumentException iae) { - LOG.error("Cannot find a matched type against from '" + typeStr + "'"); - return null; - } - } - @Override public final List<String> getAllTableNames(String databaseName) { HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; @@ -814,7 +806,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { List<FieldSchema> partitionKeys = table.getPartitionKeys(); if (partitionKeys != null && partitionKeys.size() > 0) { - org.apache.tajo.catalog.Schema expressionSchema = SchemaFactory.newV1(); + SchemaBuilder expressionSchema = SchemaBuilder.builder(); StringBuilder sb = new StringBuilder(); if (partitionKeys.size() > 0) { for (int i = 0; i < partitionKeys.size(); i++) { @@ -822,7 +814,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(fieldSchema.getType()); String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + CatalogConstants.IDENTIFIER_DELIMITER + fieldSchema.getName(); - expressionSchema.addColumn(new Column(fieldName, dataType)); + expressionSchema.add(new Column(fieldName, dataType)); if (i > 0) { sb.append(","); } @@ -833,7 +825,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { tableName, PartitionType.COLUMN, sb.toString(), - expressionSchema); + expressionSchema.build()); } } else { throw new UndefinedPartitionMethodException(tableName); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java index e8d60cf..fdb1853 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java @@ -95,15 +95,16 @@ public class TestHiveCatalogStore { public void testTableUsingTextFile() throws Exception { TableMeta meta = new TableMeta(BuiltinStorages.TEXT, new KeyValueSet()); - org.apache.tajo.catalog.Schema schema = SchemaFactory.newV1(); - schema.addColumn("c_custkey", TajoDataTypes.Type.INT4); - schema.addColumn("c_name", TajoDataTypes.Type.TEXT); - schema.addColumn("c_address", TajoDataTypes.Type.TEXT); - schema.addColumn("c_nationkey", TajoDataTypes.Type.INT4); - schema.addColumn("c_phone", TajoDataTypes.Type.TEXT); - schema.addColumn("c_acctbal", TajoDataTypes.Type.FLOAT8); - schema.addColumn("c_mktsegment", TajoDataTypes.Type.TEXT); - schema.addColumn("c_comment", TajoDataTypes.Type.TEXT); + org.apache.tajo.catalog.Schema schema = SchemaBuilder.builder() + .add("c_custkey", TajoDataTypes.Type.INT4) + .add("c_name", TajoDataTypes.Type.TEXT) + .add("c_address", TajoDataTypes.Type.TEXT) + .add("c_nationkey", TajoDataTypes.Type.INT4) + .add("c_phone", TajoDataTypes.Type.TEXT) + .add("c_acctbal", TajoDataTypes.Type.FLOAT8) + .add("c_mktsegment", TajoDataTypes.Type.TEXT) + .add("c_comment", TajoDataTypes.Type.TEXT) + .build(); TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri()); @@ -135,10 +136,11 @@ public class TestHiveCatalogStore { options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); TableMeta meta = new TableMeta(BuiltinStorages.RCFILE, options); - org.apache.tajo.catalog.Schema schema = SchemaFactory.newV1(); - schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4); - schema.addColumn("r_name", TajoDataTypes.Type.TEXT); - schema.addColumn("r_comment", TajoDataTypes.Type.TEXT); + org.apache.tajo.catalog.Schema schema = SchemaBuilder.builder() + .add("r_regionkey", TajoDataTypes.Type.INT4) + .add("r_name", TajoDataTypes.Type.TEXT) + .add("r_comment", TajoDataTypes.Type.TEXT) + .build(); TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); @@ -169,10 +171,11 @@ public class TestHiveCatalogStore { options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); TableMeta meta = new TableMeta(BuiltinStorages.RCFILE, options); - org.apache.tajo.catalog.Schema schema = SchemaFactory.newV1(); - schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4); - schema.addColumn("r_name", TajoDataTypes.Type.TEXT); - schema.addColumn("r_comment", TajoDataTypes.Type.TEXT); + org.apache.tajo.catalog.Schema schema = SchemaBuilder.builder() + .add("r_regionkey", TajoDataTypes.Type.INT4) + .add("r_name", TajoDataTypes.Type.TEXT) + .add("r_comment", TajoDataTypes.Type.TEXT) + .build(); TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); @@ -203,14 +206,15 @@ public class TestHiveCatalogStore { options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava("\u0003")); TableMeta meta = new TableMeta(BuiltinStorages.TEXT, options); - org.apache.tajo.catalog.Schema schema = SchemaFactory.newV1(); - schema.addColumn("s_suppkey", TajoDataTypes.Type.INT4); - schema.addColumn("s_name", TajoDataTypes.Type.TEXT); - schema.addColumn("s_address", TajoDataTypes.Type.TEXT); - schema.addColumn("s_nationkey", TajoDataTypes.Type.INT4); - schema.addColumn("s_phone", TajoDataTypes.Type.TEXT); - schema.addColumn("s_acctbal", TajoDataTypes.Type.FLOAT8); - schema.addColumn("s_comment", TajoDataTypes.Type.TEXT); + org.apache.tajo.catalog.Schema schema = SchemaBuilder.builder() + .add("s_suppkey", TajoDataTypes.Type.INT4) + .add("s_name", TajoDataTypes.Type.TEXT) + .add("s_address", TajoDataTypes.Type.TEXT) + .add("s_nationkey", TajoDataTypes.Type.INT4) + .add("s_phone", TajoDataTypes.Type.TEXT) + .add("s_acctbal", TajoDataTypes.Type.FLOAT8) + .add("s_comment", TajoDataTypes.Type.TEXT) + .build(); TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, SUPPLIER), schema, meta, new Path(warehousePath, new Path(DB_NAME, SUPPLIER)).toUri()); @@ -252,18 +256,20 @@ public class TestHiveCatalogStore { public void testAddTableByPartition() throws Exception { TableMeta meta = new TableMeta("TEXT", new KeyValueSet()); - org.apache.tajo.catalog.Schema schema = SchemaFactory.newV1(); - schema.addColumn("n_name", TajoDataTypes.Type.TEXT); - schema.addColumn("n_regionkey", TajoDataTypes.Type.INT4); - schema.addColumn("n_comment", TajoDataTypes.Type.TEXT); + org.apache.tajo.catalog.Schema schema = SchemaBuilder.builder() + .add("n_name", TajoDataTypes.Type.TEXT) + .add("n_regionkey", TajoDataTypes.Type.INT4) + .add("n_comment", TajoDataTypes.Type.TEXT) + .build(); TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, NATION), schema, meta, new Path(warehousePath, new Path(DB_NAME, NATION)).toUri()); - org.apache.tajo.catalog.Schema expressionSchema = SchemaFactory.newV1(); - expressionSchema.addColumn("n_nationkey", TajoDataTypes.Type.INT4); - expressionSchema.addColumn("n_date", TajoDataTypes.Type.TEXT); + org.apache.tajo.catalog.Schema expressionSchema = SchemaBuilder.builder() + .add("n_nationkey", TajoDataTypes.Type.INT4) + .add("n_date", TajoDataTypes.Type.TEXT) + .build(); PartitionMethodDesc partitions = new PartitionMethodDesc( DB_NAME, @@ -500,10 +506,11 @@ public class TestHiveCatalogStore { @Test public void testGetAllTableNames() throws Exception{ TableMeta meta = new TableMeta(BuiltinStorages.TEXT, new KeyValueSet()); - org.apache.tajo.catalog.Schema schema = SchemaFactory.newV1(); - schema.addColumn("n_name", TajoDataTypes.Type.TEXT); - schema.addColumn("n_regionkey", TajoDataTypes.Type.INT4); - schema.addColumn("n_comment", TajoDataTypes.Type.TEXT); + org.apache.tajo.catalog.Schema schema = SchemaBuilder.builder() + .add("n_name", TajoDataTypes.Type.TEXT) + .add("n_regionkey", TajoDataTypes.Type.INT4) + .add("n_comment", TajoDataTypes.Type.TEXT) + .build(); String[] tableNames = new String[]{"table1", "table2", "table3"}; @@ -528,10 +535,11 @@ public class TestHiveCatalogStore { @Test public void testDeleteTable() throws Exception { TableMeta meta = new TableMeta(BuiltinStorages.TEXT, new KeyValueSet()); - org.apache.tajo.catalog.Schema schema = SchemaFactory.newV1(); - schema.addColumn("n_name", TajoDataTypes.Type.TEXT); - schema.addColumn("n_regionkey", TajoDataTypes.Type.INT4); - schema.addColumn("n_comment", TajoDataTypes.Type.TEXT); + org.apache.tajo.catalog.Schema schema = SchemaBuilder.builder() + .add("n_name", TajoDataTypes.Type.TEXT) + .add("n_regionkey", TajoDataTypes.Type.INT4) + .add("n_comment", TajoDataTypes.Type.TEXT) + .build(); String tableName = "table1"; TableDesc table = new TableDesc(DB_NAME + "." + tableName, schema, meta, warehousePath.toUri()); @@ -553,10 +561,11 @@ public class TestHiveCatalogStore { options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); TableMeta meta = new TableMeta(BuiltinStorages.SEQUENCE_FILE, options); - org.apache.tajo.catalog.Schema schema = SchemaFactory.newV1(); - schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4); - schema.addColumn("r_name", TajoDataTypes.Type.TEXT); - schema.addColumn("r_comment", TajoDataTypes.Type.TEXT); + org.apache.tajo.catalog.Schema schema = SchemaBuilder.builder() + .add("r_regionkey", TajoDataTypes.Type.INT4) + .add("r_name", TajoDataTypes.Type.TEXT) + .add("r_comment", TajoDataTypes.Type.TEXT) + .build(); TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); @@ -587,10 +596,11 @@ public class TestHiveCatalogStore { options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); TableMeta meta = new TableMeta(BuiltinStorages.SEQUENCE_FILE, options); - org.apache.tajo.catalog.Schema schema = SchemaFactory.newV1(); - schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4); - schema.addColumn("r_name", TajoDataTypes.Type.TEXT); - schema.addColumn("r_comment", TajoDataTypes.Type.TEXT); + org.apache.tajo.catalog.Schema schema = SchemaBuilder.builder() + .add("r_regionkey", TajoDataTypes.Type.INT4) + .add("r_name", TajoDataTypes.Type.TEXT) + .add("r_comment", TajoDataTypes.Type.TEXT) + .build(); TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); @@ -619,15 +629,16 @@ public class TestHiveCatalogStore { public void testTableUsingParquet() throws Exception { TableMeta meta = new TableMeta("PARQUET", new KeyValueSet()); - org.apache.tajo.catalog.Schema schema = SchemaFactory.newV1(); - schema.addColumn("c_custkey", TajoDataTypes.Type.INT4); - schema.addColumn("c_name", TajoDataTypes.Type.TEXT); - schema.addColumn("c_address", TajoDataTypes.Type.TEXT); - schema.addColumn("c_nationkey", TajoDataTypes.Type.INT4); - schema.addColumn("c_phone", TajoDataTypes.Type.TEXT); - schema.addColumn("c_acctbal", TajoDataTypes.Type.FLOAT8); - schema.addColumn("c_mktsegment", TajoDataTypes.Type.TEXT); - schema.addColumn("c_comment", TajoDataTypes.Type.TEXT); + org.apache.tajo.catalog.Schema schema = SchemaBuilder.builder() + .add("c_custkey", TajoDataTypes.Type.INT4) + .add("c_name", TajoDataTypes.Type.TEXT) + .add("c_address", TajoDataTypes.Type.TEXT) + .add("c_nationkey", TajoDataTypes.Type.INT4) + .add("c_phone", TajoDataTypes.Type.TEXT) + .add("c_acctbal", TajoDataTypes.Type.FLOAT8) + .add("c_mktsegment", TajoDataTypes.Type.TEXT) + .add("c_comment", TajoDataTypes.Type.TEXT) + .build(); TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri()); @@ -656,18 +667,19 @@ public class TestHiveCatalogStore { TableMeta meta = new TableMeta(BuiltinStorages.TEXT, new KeyValueSet()); - org.apache.tajo.catalog.Schema schema = SchemaFactory.newV1(); - schema.addColumn("col1", TajoDataTypes.Type.INT4); - schema.addColumn("col2", TajoDataTypes.Type.INT1); - schema.addColumn("col3", TajoDataTypes.Type.INT2); - schema.addColumn("col4", TajoDataTypes.Type.INT8); - schema.addColumn("col5", TajoDataTypes.Type.BOOLEAN); - schema.addColumn("col6", TajoDataTypes.Type.FLOAT4); - schema.addColumn("col7", TajoDataTypes.Type.FLOAT8); - schema.addColumn("col8", TajoDataTypes.Type.TEXT); - schema.addColumn("col9", TajoDataTypes.Type.BLOB); - schema.addColumn("col10", TajoDataTypes.Type.TIMESTAMP); - schema.addColumn("col11", TajoDataTypes.Type.DATE); + org.apache.tajo.catalog.Schema schema = SchemaBuilder.builder() + .add("col1", TajoDataTypes.Type.INT4) + .add("col2", TajoDataTypes.Type.INT1) + .add("col3", TajoDataTypes.Type.INT2) + .add("col4", TajoDataTypes.Type.INT8) + .add("col5", TajoDataTypes.Type.BOOLEAN) + .add("col6", TajoDataTypes.Type.FLOAT4) + .add("col7", TajoDataTypes.Type.FLOAT8) + .add("col8", TajoDataTypes.Type.TEXT) + .add("col9", TajoDataTypes.Type.BLOB) + .add("col10", TajoDataTypes.Type.TIMESTAMP) + .add("col11", TajoDataTypes.Type.DATE) + .build(); TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, tableName), schema, meta, new Path(warehousePath, new Path(DB_NAME, tableName)).toUri()); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 1c93d08..88fabe2 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -32,12 +32,12 @@ import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.common.TajoDataTypes.*; +import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.*; -import org.apache.tajo.util.JavaResourceUtil; -import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.expr.AlgebraicUtil; import org.apache.tajo.plan.util.PartitionFilterAlgebraVisitor; +import org.apache.tajo.util.JavaResourceUtil; import org.apache.tajo.util.Pair; import java.io.IOException; @@ -2789,7 +2789,8 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo // Since the column names in the unified name are always sorted // in order of occurrence position in the relation schema, // they can be uniquely identified. - String unifiedName = CatalogUtil.getUnifiedSimpleColumnName(SchemaFactory.newV1(relationSchema), columnNames); + String unifiedName = CatalogUtil.getUnifiedSimpleColumnName( + SchemaBuilder.builder().addAll(relationSchema.getRootColumns()).build(), columnNames); pstmt.setInt(1, databaseId); pstmt.setInt(2, tableId); pstmt.setString(3, unifiedName); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java index 51461d1..3305801 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java @@ -197,10 +197,11 @@ public class CatalogTestingUtil { } public static TableDesc buildTableDesc(String databaseName, String tableName, String testDir) throws IOException { - Schema schema = SchemaFactory.newV1(); - schema.addColumn(CatalogUtil.buildFQName(tableName, "Column"), Type.BLOB); - schema.addColumn(CatalogUtil.buildFQName(tableName, "column"), Type.INT4); - schema.addColumn(CatalogUtil.buildFQName(tableName, "cOlumn"), Type.INT8); + Schema schema = SchemaBuilder.builder() + .add(CatalogUtil.buildFQName(tableName, "Column"), Type.BLOB) + .add(CatalogUtil.buildFQName(tableName, "column"), Type.INT4) + .add(CatalogUtil.buildFQName(tableName, "cOlumn"), Type.INT8) + .build(); Path path = new Path(testDir + "/" + UUID.randomUUID().toString(), tableName); TableDesc desc = new TableDesc( CatalogUtil.buildFQName(databaseName, tableName), @@ -213,9 +214,10 @@ public class CatalogTestingUtil { } public static TableDesc buildPartitionTableDesc(String databaseName, String tableName, String testDir) throws Exception { - Schema partSchema = SchemaFactory.newV1(); - partSchema.addColumn(CatalogUtil.buildFQName(tableName, "DaTe"), Type.TEXT); - partSchema.addColumn(CatalogUtil.buildFQName(tableName, "dAtE"), Type.TEXT); + Schema partSchema = SchemaBuilder.builder() + .add(CatalogUtil.buildFQName(tableName, "DaTe"), Type.TEXT) + .add(CatalogUtil.buildFQName(tableName, "dAtE"), Type.TEXT) + .build(); PartitionMethodDesc partitionMethodDesc = new PartitionMethodDesc(DEFAULT_DATABASE_NAME, tableName, CatalogProtos.PartitionType.COLUMN, "id,name", partSchema); http://git-wip-us.apache.org/repos/asf/tajo/blob/4aef83a3/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index c64fe43..9c1e430 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -181,10 +181,11 @@ public class TestCatalog { } private TableDesc createMockupTable(String databaseName, String tableName) throws IOException { - schema1 = SchemaFactory.newV1(); - schema1.addColumn(FieldName1, Type.BLOB); - schema1.addColumn(FieldName2, Type.INT4); - schema1.addColumn(FieldName3, Type.INT8); + schema1 = SchemaBuilder.builder() + .add(FieldName1, Type.BLOB) + .add(FieldName2, Type.INT4) + .add(FieldName3, Type.INT8) + .build(); Path path = new Path(CommonTestingUtil.getTestDir(), tableName); TableDesc table = new TableDesc( CatalogUtil.buildFQName(databaseName, tableName), @@ -299,10 +300,11 @@ public class TestCatalog { @Test public void testGetTable() throws Exception { - schema1 = SchemaFactory.newV1(); - schema1.addColumn(FieldName1, Type.BLOB); - schema1.addColumn(FieldName2, Type.INT4); - schema1.addColumn(FieldName3, Type.INT8); + schema1 = SchemaBuilder.builder() + .add(FieldName1, Type.BLOB) + .add(FieldName2, Type.INT4) + .add(FieldName3, Type.INT8) + .build(); Path path = new Path(CommonTestingUtil.getTestDir(), "table1"); TableDesc meta = new TableDesc( CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "getTable"), @@ -361,26 +363,22 @@ public class TestCatalog { // |- s8 // |- s9 - Schema nestedSchema = SchemaFactory.newV1(); - nestedSchema.addColumn("s1", Type.INT8); + SchemaBuilder nestedSchema = SchemaBuilder.builder(); + nestedSchema.add("s1", Type.INT8); + nestedSchema.add("s2", Type.INT8); - nestedSchema.addColumn("s2", Type.INT8); + Schema s5 = SchemaBuilder.builder().add("s6", Type.INT8).build(); + Schema s7 = SchemaBuilder.builder().add("s5", new TypeDesc(s5)).build(); - Schema s5 = SchemaFactory.newV1(); - s5.addColumn("s6", Type.INT8); + Schema s3 = SchemaBuilder.builder() + .add("s4", Type.INT8) + .add("s7", new TypeDesc(s7)) + .add("s8", Type.INT8).build(); - Schema s7 = SchemaFactory.newV1(); - s7.addColumn("s5", new TypeDesc(s5)); + nestedSchema.add("s3", new TypeDesc(s3)); + nestedSchema.add("s9", Type.INT8); - Schema s3 = SchemaFactory.newV1(); - s3.addColumn("s4", Type.INT8); - s3.addColumn("s7", new TypeDesc(s7)); - s3.addColumn("s8", Type.INT8); - - nestedSchema.addColumn("s3", new TypeDesc(s3)); - nestedSchema.addColumn("s9", Type.INT8); - - assertSchemaEquality("nested_schema1", nestedSchema); + assertSchemaEquality("nested_schema1", nestedSchema.build()); } @Test @@ -398,26 +396,26 @@ public class TestCatalog { // |- s3 // |- s4 - Schema nestedSchema = SchemaFactory.newV1(); - nestedSchema.addColumn("s1", Type.INT8); - - nestedSchema.addColumn("s2", Type.INT8); + SchemaBuilder nestedSchema = SchemaBuilder.builder(); + nestedSchema.add("s1", Type.INT8); + nestedSchema.add("s2", Type.INT8); - Schema s5 = SchemaFactory.newV1(); - s5.addColumn("s6", Type.INT8); + Schema s5 = SchemaBuilder.builder() + .add("s6", Type.INT8) + .build(); - Schema s7 = SchemaFactory.newV1(); - s7.addColumn("s5", new TypeDesc(s5)); + SchemaBuilder s7 = SchemaBuilder.builder(); + s7.add("s5", new TypeDesc(s5)); - Schema s3 = SchemaFactory.newV1(); - s3.addColumn("s4", Type.INT8); - s3.addColumn("s7", new TypeDesc(s7)); - s3.addColumn("s8", Type.INT8); + SchemaBuilder s3 = SchemaBuilder.builder(); + s3.add("s4", Type.INT8); + s3.add("s7", new TypeDesc(s7.build())); + s3.add("s8", Type.INT8); - nestedSchema.addColumn("s3", new TypeDesc(s3)); - nestedSchema.addColumn("s9", Type.INT8); + nestedSchema.add("s3", new TypeDesc(s3.build())); + nestedSchema.add("s9", Type.INT8); - assertSchemaEquality("nested_schema2", nestedSchema); + assertSchemaEquality("nested_schema2", nestedSchema.build()); } static IndexDesc desc1; @@ -426,11 +424,12 @@ public class TestCatalog { static Schema relationSchema; public static TableDesc prepareTable() throws IOException { - relationSchema = SchemaFactory.newV1(); - relationSchema.addColumn(DEFAULT_DATABASE_NAME + ".indexed.id", Type.INT4) - .addColumn(DEFAULT_DATABASE_NAME + ".indexed.name", Type.TEXT) - .addColumn(DEFAULT_DATABASE_NAME + ".indexed.age", Type.INT4) - .addColumn(DEFAULT_DATABASE_NAME + ".indexed.score", Type.FLOAT8); + relationSchema = SchemaBuilder.builder() + .add(DEFAULT_DATABASE_NAME + ".indexed.id", Type.INT4) + .add(DEFAULT_DATABASE_NAME + ".indexed.name", Type.TEXT) + .add(DEFAULT_DATABASE_NAME + ".indexed.age", Type.INT4) + .add(DEFAULT_DATABASE_NAME + ".indexed.score", Type.FLOAT8) + .build(); String tableName = "indexed"; @@ -625,11 +624,12 @@ public class TestCatalog { @Test public final void testAddAndDeleteTablePartitionByHash1() throws Exception { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4) - .addColumn("name", Type.TEXT) - .addColumn("age", Type.INT4) - .addColumn("score", Type.FLOAT8); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("name", Type.TEXT) + .add("age", Type.INT4) + .add("score", Type.FLOAT8) + .build(); String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); @@ -637,8 +637,9 @@ public class TestCatalog { TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); - Schema partSchema = SchemaFactory.newV1(); - partSchema.addColumn("id", Type.INT4); + Schema partSchema = SchemaBuilder.builder() + .add("id", Type.INT4) + .build(); PartitionMethodDesc partitionDesc = new PartitionMethodDesc(DEFAULT_DATABASE_NAME, tableName, @@ -665,19 +666,21 @@ public class TestCatalog { @Test public final void testAddAndDeleteTablePartitionByHash2() throws Exception { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4) - .addColumn("name", Type.TEXT) - .addColumn("age", Type.INT4) - .addColumn("score", Type.FLOAT8); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("name", Type.TEXT) + .add("age", Type.INT4) + .add("score", Type.FLOAT8) + .build(); String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); - Schema partSchema = SchemaFactory.newV1(); - partSchema.addColumn("id", Type.INT4); + Schema partSchema = SchemaBuilder.builder() + .add("id", Type.INT4) + .build(); PartitionMethodDesc partitionDesc = new PartitionMethodDesc(DEFAULT_DATABASE_NAME, tableName, CatalogProtos.PartitionType.HASH, "id", partSchema); @@ -703,19 +706,21 @@ public class TestCatalog { @Test public final void testAddAndDeleteTablePartitionByList() throws Exception { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4) - .addColumn("name", Type.TEXT) - .addColumn("age", Type.INT4) - .addColumn("score", Type.FLOAT8); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("name", Type.TEXT) + .add("age", Type.INT4) + .add("score", Type.FLOAT8) + .build(); String tableName = CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); - Schema partSchema = SchemaFactory.newV1(); - partSchema.addColumn("id", Type.INT4); + Schema partSchema = SchemaBuilder.builder() + .add("id", Type.INT4) + .build(); PartitionMethodDesc partitionDesc = new PartitionMethodDesc(DEFAULT_DATABASE_NAME, tableName, CatalogProtos.PartitionType.LIST, "id", partSchema); @@ -740,19 +745,21 @@ public class TestCatalog { @Test public final void testAddAndDeleteTablePartitionByRange() throws Exception { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4) - .addColumn("name", Type.TEXT) - .addColumn("age", Type.INT4) - .addColumn("score", Type.FLOAT8); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("name", Type.TEXT) + .add("age", Type.INT4) + .add("score", Type.FLOAT8) + .build(); String tableName = CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); - Schema partSchema = SchemaFactory.newV1(); - partSchema.addColumn("id", Type.INT4); + Schema partSchema = SchemaBuilder.builder() + .add("id", Type.INT4) + .build(); PartitionMethodDesc partitionDesc = new PartitionMethodDesc(DEFAULT_DATABASE_NAME, tableName, CatalogProtos.PartitionType.RANGE, "id", partSchema); @@ -777,11 +784,12 @@ public class TestCatalog { // TODO: This should be added at TAJO-1891 public final void testAddAndDeleteTablePartitionByColumn() throws Exception { - Schema schema = SchemaFactory.newV1(); - schema.addColumn("id", Type.INT4) - .addColumn("name", Type.TEXT) - .addColumn("age", Type.INT4) - .addColumn("score", Type.FLOAT8); + Schema schema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("name", Type.TEXT) + .add("age", Type.INT4) + .add("score", Type.FLOAT8) + .build(); String simpleTableName = "addedtable"; String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, simpleTableName); @@ -789,9 +797,10 @@ public class TestCatalog { opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); - Schema partSchema = SchemaFactory.newV1(); - partSchema.addColumn("id", Type.INT4); - partSchema.addColumn("name", Type.TEXT); + Schema partSchema = SchemaBuilder.builder() + .add("id", Type.INT4) + .add("name", Type.TEXT) + .build(); PartitionMethodDesc partitionMethodDesc = new PartitionMethodDesc(DEFAULT_DATABASE_NAME, tableName,
