TAJO-1359: Add nested field projector and language extension to project nested record. (hyunsik)
Closes #422 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/0d1bf41f Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/0d1bf41f Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/0d1bf41f Branch: refs/heads/index_support Commit: 0d1bf41f3aff119af5d95270d735101a81d5721c Parents: ddd3921 Author: Hyunsik Choi <[email protected]> Authored: Mon May 11 14:34:07 2015 -0700 Committer: Hyunsik Choi <[email protected]> Committed: Mon May 11 14:35:09 2015 -0700 ---------------------------------------------------------------------- CHANGES | 7 +- .../org/apache/tajo/catalog/CatalogUtil.java | 12 +- .../org/apache/tajo/catalog/NestedPathUtil.java | 109 ++++++ .../java/org/apache/tajo/catalog/Schema.java | 109 +++++- .../org/apache/tajo/catalog/SchemaUtil.java | 33 +- .../tajo/catalog/store/HiveCatalogStore.java | 4 +- .../cli/tsql/commands/DescTableCommand.java | 4 +- .../org/apache/tajo/storage/RowStoreUtil.java | 2 +- .../java/org/apache/tajo/storage/Tuple.java | 2 +- .../java/org/apache/tajo/util/StringUtils.java | 70 ++++ .../main/java/org/apache/tajo/util/TUtil.java | 32 -- .../org/apache/tajo/engine/parser/SQLParser.g4 | 2 +- .../apache/tajo/engine/parser/SQLAnalyzer.java | 17 +- .../engine/planner/PhysicalPlannerImpl.java | 17 +- .../tajo/engine/planner/global/DataChannel.java | 4 +- .../engine/planner/global/GlobalPlanner.java | 6 +- .../global/builder/DistinctGroupbyBuilder.java | 6 +- .../DistinctGroupbyThirdAggregationExec.java | 2 +- .../planner/physical/PhysicalPlanUtil.java | 2 +- .../engine/planner/physical/SeqScanExec.java | 27 +- .../tajo/engine/utils/TupleCacheScanner.java | 2 +- .../org/apache/tajo/engine/utils/TupleUtil.java | 8 +- .../exec/NonForwardQueryResultFileScanner.java | 2 +- .../NonForwardQueryResultSystemScanner.java | 20 +- .../resources/webapps/admin/catalogview.jsp | 5 +- .../engine/function/TestFunctionLoader.java | 5 +- .../tajo/engine/planner/TestLogicalPlanner.java | 8 +- .../apache/tajo/engine/query/TestCTASQuery.java | 4 +- .../tajo/engine/query/TestJoinBroadcast.java | 2 +- .../engine/query/TestSelectNestedRecord.java | 71 ++++ .../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 2 +- .../rs/resources/TestQueryResultResource.java | 6 +- .../TestSelectNestedRecord/sample1/table.json | 3 + .../TestSelectNestedRecord/tweets/sample1.json | 4 + .../TestSelectNestedRecord/sample1_ddl.sql | 7 + .../TestSelectNestedRecord/sample2_ddl.sql | 19 ++ .../testNestedFieldAsGroupbyKey1.sql | 7 + .../testNestedFieldAsJoinKey1.sql | 7 + .../TestSelectNestedRecord/testSelect1.sql | 1 + .../TestSelectNestedRecord/testSelect2.sql | 61 ++++ .../TestSelectNestedRecord/tweets_ddl.sql | 74 +++++ .../testSelectWithParentheses2.sql | 2 +- .../testNestedFieldAsGroupbyKey1.result | 6 + .../testNestedFieldAsJoinKey1.result | 6 + .../TestSelectNestedRecord/testSelect1.result | 5 + .../TestSelectNestedRecord/testSelect2.result | 6 + .../apache/tajo/jdbc/TajoDatabaseMetaData.java | 2 +- .../tajo/plan/LogicalPlanPreprocessor.java | 4 +- .../org/apache/tajo/plan/LogicalPlanner.java | 32 +- .../apache/tajo/plan/expr/RowConstantEval.java | 3 +- .../tajo/plan/expr/WindowFunctionEval.java | 3 +- .../function/python/PythonScriptEngine.java | 12 +- .../GreedyHeuristicJoinOrderAlgorithm.java | 8 +- .../apache/tajo/plan/joinorder/JoinEdge.java | 4 +- .../apache/tajo/plan/joinorder/JoinGraph.java | 6 +- .../tajo/plan/logical/DistinctGroupbyNode.java | 4 +- .../apache/tajo/plan/logical/EvalExprNode.java | 3 +- .../apache/tajo/plan/logical/GroupbyNode.java | 10 +- .../tajo/plan/logical/ProjectionNode.java | 3 +- .../apache/tajo/plan/logical/RelationNode.java | 12 + .../tajo/plan/logical/ShuffleFileWriteNode.java | 3 +- .../tajo/plan/logical/TruncateTableNode.java | 4 +- .../apache/tajo/plan/logical/WindowAggNode.java | 7 +- .../tajo/plan/nameresolver/NameResolver.java | 194 ++++++++--- .../plan/nameresolver/ResolverByLegacy.java | 8 +- .../plan/rewrite/rules/FilterPushDownRule.java | 4 +- .../rewrite/rules/PartitionedTableRewriter.java | 4 +- .../org/apache/tajo/plan/util/PlannerUtil.java | 2 +- .../tajo/plan/verifier/LogicalPlanVerifier.java | 24 +- .../plan/verifier/PreLogicalPlanVerifier.java | 6 +- .../org/apache/tajo/storage/MergeScanner.java | 2 +- .../org/apache/tajo/storage/RowStoreUtil.java | 2 +- .../java/org/apache/tajo/storage/Scanner.java | 9 + .../org/apache/tajo/storage/StorageManager.java | 4 +- .../storage/hbase/AbstractHBaseAppender.java | 2 - .../tajo/storage/hbase/ColumnMapping.java | 2 +- .../HBaseBinarySerializerDeserializer.java | 10 +- .../apache/tajo/storage/hbase/HBaseScanner.java | 22 +- .../tajo/storage/hbase/HBaseStorageManager.java | 6 +- .../java/org/apache/tajo/storage/CSVFile.java | 2 +- .../org/apache/tajo/storage/FileScanner.java | 2 +- .../apache/tajo/storage/FileStorageManager.java | 2 +- .../apache/tajo/storage/avro/AvroScanner.java | 25 +- .../tajo/storage/json/JsonLineDeserializer.java | 331 +++++++++++-------- .../apache/tajo/storage/json/JsonLineSerDe.java | 5 +- .../tajo/storage/json/JsonLineSerializer.java | 2 - .../storage/parquet/TajoRecordConverter.java | 7 +- .../org/apache/tajo/storage/rcfile/RCFile.java | 10 +- .../sequencefile/SequenceFileScanner.java | 16 +- .../tajo/storage/text/CSVLineDeserializer.java | 26 +- .../apache/tajo/storage/text/CSVLineSerDe.java | 5 +- .../tajo/storage/text/DelimitedTextFile.java | 11 +- .../tajo/storage/text/TextLineDeserializer.java | 4 +- .../apache/tajo/storage/text/TextLineSerDe.java | 3 +- .../apache/tajo/storage/TestMergeScanner.java | 29 +- .../org/apache/tajo/storage/TestStorages.java | 35 +- 96 files changed, 1279 insertions(+), 467 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 0aad306..84be66a 100644 --- a/CHANGES +++ b/CHANGES @@ -27,8 +27,8 @@ Release 0.11.0 - unreleased TAJO-1452: Improve function listing order (Contributed Dongjoon Hyun, Committed by hyunsik) - TAJO-1576: Sometimes DefaultTajoCliOutputFormatter.parseErrorMessage() eliminates - an important kind of information. + TAJO-1576: Sometimes DefaultTajoCliOutputFormatter.parseErrorMessage() + eliminates an important kind of information. (Contributed by Jongyoung Park, Committed by jihoon) TAJO-1408: Make IntermediateEntryProto more compact. @@ -277,6 +277,9 @@ Release 0.11.0 - unreleased SUB TASKS + TAJO-1359: Add nested field projector and language extension to project + nested record. (hyunsik) + TAJO-1529: Implement json_extract_path_text(string, string) function. (jinho) http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 12f36b1..8e5b657 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -171,6 +171,16 @@ public class CatalogUtil { return openQuote && closeQuote; } + /** + * True if a given name is a simple identifier, meaning is not a dot-chained name. + * + * @param columnOrTableName Column or Table name to be checked + * @return True if a given name is a simple identifier. Otherwise, it will return False. + */ + public static boolean isSimpleIdentifier(String columnOrTableName) { + return columnOrTableName.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP).length == 1; + } + public static boolean isFQColumnName(String tableName) { return tableName.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP).length == 3; } @@ -662,7 +672,7 @@ public class CatalogUtil { if (types[i].getType() != Type.NULL_TYPE) { Type candidate = TUtil.getFromNestedMap(OPERATION_CASTING_MAP, widest.getType(), types[i].getType()); if (candidate == null) { - throw new InvalidOperationException("No matched operation for those types: " + TUtil.arrayToString + throw new InvalidOperationException("No matched operation for those types: " + StringUtils.join (types)); } widest = newSimpleDataType(candidate); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java new file mode 100644 index 0000000..58b4f26 --- /dev/null +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog; + +import com.google.common.base.Preconditions; +import org.apache.tajo.common.TajoDataTypes.Type; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Utility methods for nested field + */ +public class NestedPathUtil { + public static final String PATH_DELIMITER = "/"; + + public static final List<String> ROOT_PATH = Collections.unmodifiableList(new ArrayList<String>()); + + public static boolean isPath(String name) { + return name.indexOf(PATH_DELIMITER.charAt(0)) >= 0; + } + + public static String makePath(String[] parts) { + return makePath(parts, 0); + } + + public static String makePath(String[] parts, int startIndex) { + return makePath(parts, startIndex, parts.length); + } + + /** + * Make a nested field path + * + * @param parts path parts + * @param startIndex startIndex + * @param depth Depth + * @return Path + */ + public static String makePath(String[] parts, int startIndex, int depth) { + Preconditions.checkArgument(startIndex <= (parts.length - 1)); + + StringBuilder sb = new StringBuilder(); + for (int i = startIndex; i < depth; i++) { + sb.append(PATH_DELIMITER); + sb.append(parts[i].toString()); + } + + return sb.toString(); + } + + /** + * Lookup the actual column corresponding to a given path. + * We assume that a path starts with the slash '/' and it + * does not include the root field. + * + * @param nestedField Nested column + * @param path Path which starts with '/'; + * @return Column corresponding to the path + */ + public static Column lookupPath(Column nestedField, String path) { + Preconditions.checkArgument(path.charAt(0) == PATH_DELIMITER.charAt(0), + "A nested field path must start with slash '/'."); + + // We assume that path starts with '/', causing an empty string "" at 0 in the path splits. + // So, we should start the index from 1 instead of 0. + return lookupPath(nestedField, path.split(PATH_DELIMITER)); + } + + public static Column lookupPath(Column nestedField, String [] paths) { + // We assume that path starts with '/', causing an empty string "" at 0 in the path splits. + // So, we should start the index from 1 instead of 0. + return lookupColumnInternal(nestedField, paths, 1); + } + + private static Column lookupColumnInternal(Column currentColumn, String [] paths, int depth) { + Column found = null; + + if (currentColumn.getDataType().getType() == Type.RECORD) { + found = currentColumn.typeDesc.nestedRecordSchema.getColumn(paths[depth]); + } + + if (found != null) { + if (found.getDataType().getType() == Type.RECORD) { + return lookupColumnInternal(found, paths, depth + 1); + } else { + return found; + } + } else { + throw new NoSuchFieldError(makePath(paths)); + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java index 054cc2c..0e4b741 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java @@ -32,6 +32,7 @@ import org.apache.tajo.common.ProtoObject; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.json.GsonObject; +import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; import java.util.*; @@ -135,7 +136,8 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject { * @param qualifier The qualifier */ public void setQualifier(String qualifier) { - List<Column> columns = getColumns(); + // only change root fields, and must keep each nested field simple name + List<Column> columns = getRootColumns(); fields.clear(); fieldsByQualifiedName.clear(); @@ -180,14 +182,39 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject { * @return The column matched to a given column name. */ public Column getColumn(String name) { - String [] parts = name.split("\\."); - // Some of the string can includes database name and table name and column name. - // For example, it can be 'default.table1.id'. - // Therefore, spilt string array length can be 3. - if (parts.length >= 2) { - return getColumnByQName(name); + + if (NestedPathUtil.isPath(name)) { + + // TODO - to be refactored + if (fieldsByQualifiedName.containsKey(name)) { + Column flattenColumn = fields.get(fieldsByQualifiedName.get(name)); + if (flattenColumn != null) { + return flattenColumn; + } + } + + String [] paths = name.split(NestedPathUtil.PATH_DELIMITER); + Column column = getColumn(paths[0]); + if (column == null) { + return null; + } + Column actualColumn = NestedPathUtil.lookupPath(column, paths); + + Column columnPath = new Column( + column.getQualifiedName() + NestedPathUtil.makePath(paths, 1), + actualColumn.typeDesc); + + return columnPath; } else { - return getColumnByName(name); + String[] parts = name.split("\\."); + // Some of the string can includes database name and table name and column name. + // For example, it can be 'default.table1.id'. + // Therefore, spilt string array length can be 3. + if (parts.length >= 2) { + return getColumnByQName(name); + } else { + return getColumnByName(name); + } } } @@ -268,12 +295,46 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject { } return -1; } - - public List<Column> getColumns() { + + /** + * Get root columns, meaning all columns except for nested fields. + * + * @return A list of root columns + */ + public List<Column> getRootColumns() { return ImmutableList.copyOf(fields); } + /** + * Get all columns, including all nested fields + * + * @return A list of all columns + */ + public List<Column> getAllColumns() { + final List<Column> columnList = TUtil.newList(); + + SchemaUtil.visitSchema(this, new ColumnVisitor() { + @Override + public void visit(int depth, List<String> path, Column column) { + if (path.size() > 0) { + String parentPath = StringUtils.join(path, NestedPathUtil.PATH_DELIMITER); + String currentPath = parentPath + NestedPathUtil.PATH_DELIMITER + column.getSimpleName(); + columnList.add(new Column(currentPath, column.getTypeDesc())); + } else { + columnList.add(column); + } + } + }); + + return columnList; + } + public boolean contains(String name) { + // TODO - It's a hack + if (NestedPathUtil.isPath(name)) { + return (getColumn(name) != null); + } + if (fieldsByQualifiedName.containsKey(name)) { return true; } @@ -288,6 +349,11 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject { } public boolean contains(Column column) { + // TODO - It's a hack + if (NestedPathUtil.isPath(column.getQualifiedName())) { + return (getColumn(column.getQualifiedName()) != null); + } + if (column.hasQualifier()) { return fieldsByQualifiedName.containsKey(column.getQualifiedName()); } else { @@ -314,7 +380,24 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject { } public boolean containsAll(Collection<Column> columns) { - return fields.containsAll(columns); + boolean containFlag = true; + + for (Column c :columns) { + if (NestedPathUtil.isPath(c.getSimpleName())) { + if (contains(c.getQualifiedName())) { + containFlag &= true; + } else { + String[] paths = c.getQualifiedName().split("/"); + boolean existRootPath = contains(paths[0]); + boolean existLeafPath = getColumn(c.getSimpleName()) != null; + containFlag &= existRootPath && existLeafPath; + } + } else { + containFlag &= fields.contains(c); + } + } + + return containFlag; } public synchronized Schema addColumn(String name, TypeDesc typeDesc) { @@ -351,7 +434,7 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject { } public synchronized void addColumns(Schema schema) { - for(Column column : schema.getColumns()) { + for(Column column : schema.getRootColumns()) { addColumn(column); } } @@ -396,7 +479,7 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject { } @Override - public void visit(int depth, Column column) { + public void visit(int depth, List<String> path, Column column) { if (column.getDataType().getType() == Type.RECORD) { DataType.Builder updatedType = DataType.newBuilder(column.getDataType()); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java index f2bb71c..c6b2f69 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java @@ -18,6 +18,10 @@ package org.apache.tajo.catalog; +import org.apache.tajo.util.TUtil; + +import java.util.List; + import static org.apache.tajo.common.TajoDataTypes.DataType; import static org.apache.tajo.common.TajoDataTypes.Type; @@ -34,12 +38,12 @@ public class SchemaUtil { static int tmpColumnSeq = 0; public static Schema merge(Schema left, Schema right) { Schema merged = new Schema(); - for(Column col : left.getColumns()) { + for(Column col : left.getRootColumns()) { if (!merged.containsByQualifiedName(col.getQualifiedName())) { merged.addColumn(col); } } - for(Column col : right.getColumns()) { + for(Column col : right.getRootColumns()) { if (merged.containsByQualifiedName(col.getQualifiedName())) { merged.addColumn("?fake" + (tmpColumnSeq++), col.getDataType()); } else { @@ -59,7 +63,7 @@ public class SchemaUtil { */ public static Schema getNaturalJoinColumns(Schema left, Schema right) { Schema common = new Schema(); - for (Column outer : left.getColumns()) { + for (Column outer : left.getRootColumns()) { if (!common.containsByName(outer.getSimpleName()) && right.containsByName(outer.getSimpleName())) { common.addColumn(new Column(outer.getSimpleName(), outer.getDataType())); } @@ -113,7 +117,7 @@ public class SchemaUtil { * Column visitor interface */ public static interface ColumnVisitor { - public void visit(int depth, Column column); + public void visit(int depth, List<String> path, Column column); } /** @@ -122,8 +126,8 @@ public class SchemaUtil { * @param function */ public static void visitSchema(Schema schema, ColumnVisitor function) { - for(Column col : schema.getColumns()) { - visitInDepthFirstOrder(0, function, col); + for(Column col : schema.getRootColumns()) { + visitInDepthFirstOrder(0, NestedPathUtil.ROOT_PATH, function, col); } } @@ -134,14 +138,21 @@ public class SchemaUtil { * @param function Visitor * @param column Current visiting column */ - private static void visitInDepthFirstOrder(int depth, ColumnVisitor function, Column column) { + private static void visitInDepthFirstOrder(int depth, + final List<String> path, + ColumnVisitor function, + Column column) { + if (column.getDataType().getType() == Type.RECORD) { - for (Column nestedColumn : column.typeDesc.nestedRecordSchema.getColumns()) { - visitInDepthFirstOrder(depth + 1, function, nestedColumn); + for (Column nestedColumn : column.typeDesc.nestedRecordSchema.getRootColumns()) { + List<String> newPath = TUtil.newList(path); + newPath.add(column.getQualifiedName()); + + visitInDepthFirstOrder(depth + 1, newPath, function, nestedColumn); } - function.visit(depth, column); + function.visit(depth, path, column); } else { - function.visit(depth, column); + function.visit(depth, path, column); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 5b1a996..6acdbd1 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -453,7 +453,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { } // set column information - List<Column> columns = tableDesc.getSchema().getColumns(); + List<Column> columns = tableDesc.getSchema().getRootColumns(); ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(columns.size()); for (Column eachField : columns) { @@ -465,7 +465,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { // set partition keys if (tableDesc.hasPartition() && tableDesc.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) { List<FieldSchema> partitionKeys = new ArrayList<FieldSchema>(); - for (Column eachPartitionKey : tableDesc.getPartitionMethod().getExpressionSchema().getColumns()) { + for (Column eachPartitionKey : tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns()) { partitionKeys.add(new FieldSchema(eachPartitionKey.getSimpleName(), HiveCatalogUtil.getHiveFieldType(eachPartitionKey.getDataType()), "")); } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java ---------------------------------------------------------------------- diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java index a3960e6..a5b53f2 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java @@ -27,7 +27,7 @@ import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.cli.tsql.TajoCli; import org.apache.tajo.util.FileUtil; -import org.apache.tajo.util.TUtil; +import org.apache.tajo.util.StringUtils; import java.util.List; import java.util.Map; @@ -126,7 +126,7 @@ public class DescTableCommand extends TajoShellCommand { sb.append("type:").append(partition.getPartitionType().name()).append("\n"); sb.append("columns:").append(":"); - sb.append(TUtil.arrayToString(partition.getExpressionSchema().toArray())); + sb.append(StringUtils.join(partition.getExpressionSchema().toArray())); } return sb.toString(); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java b/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java index 385f99c..6e16095 100644 --- a/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java +++ b/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java @@ -36,7 +36,7 @@ public class RowStoreUtil { public static int[] getTargetIds(Schema inSchema, Schema outSchema) { int[] targetIds = new int[outSchema.size()]; int i = 0; - for (Column target : outSchema.getColumns()) { + for (Column target : outSchema.getRootColumns()) { targetIds[i] = inSchema.getColumnId(target.getQualifiedName()); i++; } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java b/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java index aec784f..fce10a3 100644 --- a/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java +++ b/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java @@ -41,8 +41,8 @@ public interface Tuple extends Cloneable { void put(Datum[] values); + Datum get(int fieldId); - void setOffset(long offset); long getOffset(); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java index d035e4a..0a16072 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java @@ -371,4 +371,74 @@ public class StringUtils { return resultArray; } + + /** + * Concatenate all objects' string with a delimiter string + * + * @param objects Iterable objects + * @param delimiter Delimiter string + * @return A joined string + */ + public static String join(Iterable objects, String delimiter) { + boolean first = true; + StringBuilder sb = new StringBuilder(); + for(Object object : objects) { + if (first) { + first = false; + } else { + sb.append(delimiter); + } + + sb.append(object.toString()); + } + + return sb.toString(); + } + + /** + * Concatenate all objects' string with the delimiter ", " + * + * @param objects Iterable objects + * @return A joined string + */ + public static String join(Object[] objects) { + return join(objects, ", ", 0, objects.length); + } + + /** + * Concatenate all objects' string with a delimiter string + * + * @param objects object array + * @param delimiter Delimiter string + * @param startIndex the begin index to join + * @return A joined string + */ + public static String join(Object[] objects, String delimiter, int startIndex) { + return join(objects, delimiter, startIndex, objects.length); + } + + /** + * Concatenate all objects' string with a delimiter string + * + * @param objects object array + * @param delimiter Delimiter string + * @param startIndex the begin index to join + * @param length the number of columns to be joined + * @return A joined string + */ + public static String join(Object[] objects, String delimiter, int startIndex, int length) { + boolean first = true; + StringBuilder sb = new StringBuilder(); + for(int i = startIndex; i + startIndex < length; i++) { + if (first) { + first = false; + } else { + sb.append(delimiter); + } + + sb.append(objects[i].toString()); + } + + return sb.toString(); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java index 75f3e2a..2293ef5 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java @@ -240,38 +240,6 @@ public class TUtil { } } - public static String collectionToString(Collection objects, String delimiter) { - boolean first = true; - StringBuilder sb = new StringBuilder(); - for(Object object : objects) { - if (first) { - first = false; - } else { - sb.append(delimiter); - } - - sb.append(object.toString()); - } - - return sb.toString(); - } - - public static String arrayToString(Object [] objects) { - boolean first = true; - StringBuilder sb = new StringBuilder(); - for(Object object : objects) { - if (first) { - first = false; - } else { - sb.append(", "); - } - - sb.append(object.toString()); - } - - return sb.toString(); - } - public static <T> T [] toArray(Collection<T> collection, Class<T> type) { T array = (T) Array.newInstance(type, collection.size()); return collection.toArray((T[]) array); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index 1a63470..3ab11bd 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -1307,7 +1307,7 @@ set_qualifier ; column_reference - : ((db_name = identifier DOT)? (tb_name=identifier DOT))? name=identifier + : identifier (DOT identifier)* ; as_clause http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index 6d32fa5..7c99868 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -1054,14 +1054,17 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> { @Override public ColumnReferenceExpr visitColumn_reference(SQLParser.Column_referenceContext ctx) { - ColumnReferenceExpr column = new ColumnReferenceExpr(ctx.name.getText()); - if (checkIfExist(ctx.db_name)) { - column.setQualifier(CatalogUtil.buildFQName(ctx.db_name.getText(), ctx.tb_name.getText())); - } else if (ctx.tb_name != null) { - column.setQualifier(ctx.tb_name.getText()); + String columnReferenceName = ctx.getText(); + // find the last dot (.) position to separate a name into both a qualifier and name + int lastDotIdx = columnReferenceName.lastIndexOf("."); + + if (lastDotIdx > 0) { // if any qualifier is given + String qualifier = columnReferenceName.substring(0, lastDotIdx); + String name = columnReferenceName.substring(lastDotIdx + 1, columnReferenceName.length()); + return new ColumnReferenceExpr(qualifier, name); + } else { + return new ColumnReferenceExpr(ctx.getText()); } - - return column; } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index f132793..506b03e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -55,6 +55,7 @@ import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.IndexUtil; +import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.TaskAttemptContext; @@ -275,7 +276,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner { LOG.info(String.format("[%s] the volume of %s relations (%s) is %s and is %sfit to main maemory.", context.getTaskId().toString(), (left ? "Left" : "Right"), - TUtil.arrayToString(lineage), + StringUtils.join(lineage), FileUtil.humanReadableByteCount(volume, false), (inMemoryInnerJoinFlag ? "" : "not "))); return inMemoryInnerJoinFlag; @@ -398,18 +399,18 @@ public class PhysicalPlannerImpl implements PhysicalPlanner { larger = right; LOG.info(String.format("[%s] Left relations %s (%s) is smaller than Right relations %s (%s).", context.getTaskId().toString(), - TUtil.arrayToString(leftLineage), + StringUtils.join(leftLineage), FileUtil.humanReadableByteCount(leftSize, false), - TUtil.arrayToString(rightLineage), + StringUtils.join(rightLineage), FileUtil.humanReadableByteCount(rightSize, false))); } else { smaller = right; larger = left; LOG.info(String.format("[%s] Right relations %s (%s) is smaller than Left relations %s (%s).", context.getTaskId().toString(), - TUtil.arrayToString(rightLineage), + StringUtils.join(rightLineage), FileUtil.humanReadableByteCount(rightSize, false), - TUtil.arrayToString(leftLineage), + StringUtils.join(leftLineage), FileUtil.humanReadableByteCount(leftSize, false))); } @@ -858,7 +859,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner { } else if (storeTableNode.getType() == NodeType.CREATE_TABLE) { int i = 0; for (int j = 0; j < partitionKeyColumns.length; j++) { - int id = storeTableNode.getOutSchema().getColumns().size() + j; + int id = storeTableNode.getOutSchema().getRootColumns().size() + j; Column column = storeTableNode.getInSchema().getColumn(id); sortSpecs[i++] = new SortSpec(column, true, false); } @@ -1002,7 +1003,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner { sortNode.setInSchema(subOp.getSchema()); sortNode.setOutSchema(subOp.getSchema()); ExternalSortExec sortExec = new ExternalSortExec(ctx, sortNode, subOp); - LOG.info("The planner chooses [Sort Aggregation] in (" + TUtil.arrayToString(sortSpecs) + ")"); + LOG.info("The planner chooses [Sort Aggregation] in (" + StringUtils.join(sortSpecs) + ")"); return new SortAggregateExec(ctx, groupbyNode, sortExec); } @@ -1043,7 +1044,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner { sortNode.setInSchema(subOp.getSchema()); sortNode.setOutSchema(subOp.getSchema()); child = new ExternalSortExec(context, sortNode, subOp); - LOG.info("The planner chooses [Sort Aggregation] in (" + TUtil.arrayToString(sortSpecs) + ")"); + LOG.info("The planner chooses [Sort Aggregation] in (" + StringUtils.join(sortSpecs) + ")"); } return new WindowAggExec(context, windowAggNode, child); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java index 11548d3..ba1b0bf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java @@ -23,7 +23,7 @@ import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SchemaUtil; -import org.apache.tajo.util.TUtil; +import org.apache.tajo.util.StringUtils; import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import static org.apache.tajo.ipc.TajoWorkerProtocol.*; @@ -193,7 +193,7 @@ public class DataChannel { sb.append(" (type=").append(shuffleType); if (hasShuffleKeys()) { sb.append(", key="); - sb.append(TUtil.arrayToString(shuffleKeys)); + sb.append(StringUtils.join(shuffleKeys)); sb.append(", num=").append(numOutputs); } sb.append(")"); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index 36bdf21..ff1955e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -1146,11 +1146,11 @@ public class GlobalPlanner { Column[] shuffleKeys = new Column[partitionMethod.getExpressionSchema().size()]; int i = 0, id = 0; - for (Column column : partitionMethod.getExpressionSchema().getColumns()) { + for (Column column : partitionMethod.getExpressionSchema().getRootColumns()) { if (node.getType() == NodeType.INSERT) { id = tableSchema.getColumnId(column.getQualifiedName()); } else { - id = tableSchema.getColumns().size() + i; + id = tableSchema.getRootColumns().size() + i; } shuffleKeys[i++] = projectedSchema.getColumn(id); } @@ -1493,7 +1493,7 @@ public class GlobalPlanner { addedTableSubQueries.add(copy); //Find a SubQueryNode which contains all columns in InputSchema matched with Target and OutputSchema's column - if (copy.getInSchema().containsAll(copy.getOutSchema().getColumns())) { + if (copy.getInSchema().containsAll(copy.getOutSchema().getRootColumns())) { for (Target eachTarget : copy.getTargets()) { Set<Column> columns = EvalTreeUtil.findUniqueColumns(eachTarget.getEvalTree()); if (copy.getInSchema().containsAll(columns)) { http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java index b5e9104..13ed99b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java @@ -81,13 +81,13 @@ public class DistinctGroupbyBuilder { // If there is not grouping column, we can't find column alias. // Thus we should find the alias at Groupbynode output schema. if (groupbyNode.getGroupingColumns().length == 0 - && aggFunctions.length == groupbyNode.getOutSchema().getColumns().size()) { + && aggFunctions.length == groupbyNode.getOutSchema().getRootColumns().size()) { aggFunctions[i].setAlias(groupbyNode.getOutSchema().getColumn(i).getQualifiedName()); } } if (groupbyNode.getGroupingColumns().length == 0 - && aggFunctions.length == groupbyNode.getOutSchema().getColumns().size()) { + && aggFunctions.length == groupbyNode.getOutSchema().getRootColumns().size()) { groupbyNode.setAggFunctions(aggFunctions); } @@ -672,7 +672,7 @@ public class DistinctGroupbyBuilder { int index = 0; for(GroupbyNode eachNode: secondStageDistinctNode.getSubPlans()) { eachNode.setInSchema(firstStageDistinctNode.getOutSchema()); - for (Column column: eachNode.getOutSchema().getColumns()) { + for (Column column: eachNode.getOutSchema().getRootColumns()) { if (secondStageInSchema.getColumn(column) == null) { secondStageInSchema.addColumn(column); } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java index e71976c..7c38d36 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java @@ -110,7 +110,7 @@ public class DistinctGroupbyThirdAggregationExec extends UnaryPhysicalExec { } int index = 0; - for (Column eachOutputColumn: outSchema.getColumns()) { + for (Column eachOutputColumn: outSchema.getRootColumns()) { // If column is avg aggregation function, outschema's column type is float // but groupbyResultTupleIndex's column type is protobuf http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java index 247b373..7694b2b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java @@ -77,7 +77,7 @@ public class PhysicalPlanUtil { //In the case of partitioned table, we should return same partition key data files. int partitionDepth = 0; if (tableDesc.hasPartition()) { - partitionDepth = tableDesc.getPartitionMethod().getExpressionSchema().getColumns().size(); + partitionDepth = tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns().size(); } List<FileStatus> nonZeroLengthFiles = new ArrayList<FileStatus>(); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java index ff9477f..3d95068 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java @@ -138,6 +138,8 @@ public class SeqScanExec extends ScanExec { public void init() throws IOException { Schema projected; + // in the case where projected column or expression are given + // the target can be an empty list. if (plan.hasTargets()) { projected = new Schema(); Set<Column> columnSet = new HashSet<Column>(); @@ -150,12 +152,15 @@ public class SeqScanExec extends ScanExec { columnSet.addAll(EvalTreeUtil.findUniqueColumns(t.getEvalTree())); } - for (Column column : inSchema.getColumns()) { + for (Column column : inSchema.getAllColumns()) { if (columnSet.contains(column)) { projected.addColumn(column); } } + } else { + // no any projected columns, meaning that all columns should be projected. + // TODO - this implicit rule makes code readability bad. So, we should remove it later projected = outSchema; } @@ -163,7 +168,11 @@ public class SeqScanExec extends ScanExec { super.init(); if (plan.hasQual()) { - qual.bind(context.getEvalContext(), inSchema); + if (scanner.isProjectable()) { + qual.bind(context.getEvalContext(), projected); + } else { + qual.bind(context.getEvalContext(), inSchema); + } } } @@ -175,7 +184,7 @@ public class SeqScanExec extends ScanExec { } private void initScanner(Schema projected) throws IOException { - this.projector = new Projector(context, inSchema, outSchema, plan.getTargets()); + TableMeta meta; try { meta = (TableMeta) plan.getTableDesc().getMeta().clone(); @@ -186,6 +195,7 @@ public class SeqScanExec extends ScanExec { // set system default properties PlannerUtil.applySystemDefaultToTableProperties(context.getQueryContext(), meta); + // Why we should check nullity? See https://issues.apache.org/jira/browse/TAJO-1422 if (fragments != null) { if (fragments.length > 1) { this.scanner = new MergeScanner(context.getConf(), plan.getPhysicalSchema(), meta, @@ -198,6 +208,17 @@ public class SeqScanExec extends ScanExec { plan.getPhysicalSchema(), fragments[0], projected); } scanner.init(); + + // See Scanner.isProjectable() method Depending on the result of isProjectable(), + // the width of retrieved tuple is changed. + // + // If TRUE, the retrieved tuple will contain only projected fields. + // If FALSE, the retrieved tuple will contain projected fields and NullDatum for non-projected fields. + if (scanner.isProjectable()) { + this.projector = new Projector(context, projected, outSchema, plan.getTargets()); + } else { + this.projector = new Projector(context, inSchema, outSchema, plan.getTargets()); + } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java index ba25172..0fd2fbe 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java @@ -73,7 +73,7 @@ public class TupleCacheScanner implements Scanner { @Override public boolean isProjectable() { - return true; + return false; } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java index 3a0a1c7..c01900a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java @@ -89,7 +89,7 @@ public class TupleUtil { } int i = 0; - for (Column col : sortSchema.getColumns()) { + for (Column col : sortSchema.getRootColumns()) { ColumnStats columnStat = statMap.get(col); if (columnStat == null) { continue; @@ -121,7 +121,7 @@ public class TupleUtil { statSet.put(stat.getColumn(), stat); } - for (Column col : target.getColumns()) { + for (Column col : target.getRootColumns()) { Preconditions.checkState(statSet.containsKey(col), "ERROR: Invalid Column Stats (column stats: " + colStats + ", there exists not target " + col); } @@ -134,7 +134,7 @@ public class TupleUtil { // In outer join, empty table could be searched. // As a result, min value and max value would be null. // So, we should put NullDatum for this case. - for (Column col : target.getColumns()) { + for (Column col : target.getRootColumns()) { if (sortSpecs[sortSpecIndex].isAscending()) { if (statSet.get(col).getMinValue() != null) startTuple.put(i, statSet.get(col).getMinValue()); @@ -169,7 +169,7 @@ public class TupleUtil { else endTuple.put(i, DatumFactory.createNullDatum()); } - if (target.getColumns().size() == sortSpecs.length) { + if (target.getRootColumns().size() == sortSpecs.length) { // Not composite column sort sortSpecIndex++; } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java index 804821b..9c0bd48 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java @@ -89,7 +89,7 @@ public class NonForwardQueryResultFileScanner implements NonForwardQueryResultSc StringBuffer path = new StringBuffer(); int depth = 0; if (tableDesc.hasPartition()) { - for (Column c : tableDesc.getPartitionMethod().getExpressionSchema().getColumns()) { + for (Column c : tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns()) { String partitionValue = EvalTreeUtil.getPartitionValue(scanNode.getQual(), c.getSimpleName()); if (partitionValue == null) break; http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java index 958c252..6c9b485 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java @@ -144,7 +144,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult private List<Tuple> getTablespaces(Schema outSchema) { List<TablespaceProto> tablespaces = masterContext.getCatalog().getAllTablespaces(); List<Tuple> tuples = new ArrayList<Tuple>(tablespaces.size()); - List<Column> columns = outSchema.getColumns(); + List<Column> columns = outSchema.getRootColumns(); Tuple aTuple; for (TablespaceProto tablespace: tablespaces) { @@ -179,7 +179,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult private List<Tuple> getDatabases(Schema outSchema) { List<DatabaseProto> databases = masterContext.getCatalog().getAllDatabases(); List<Tuple> tuples = new ArrayList<Tuple>(databases.size()); - List<Column> columns = outSchema.getColumns(); + List<Column> columns = outSchema.getRootColumns(); Tuple aTuple; for (DatabaseProto database: databases) { @@ -209,7 +209,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult private List<Tuple> getTables(Schema outSchema) { List<TableDescriptorProto> tables = masterContext.getCatalog().getAllTables(); List<Tuple> tuples = new ArrayList<Tuple>(tables.size()); - List<Column> columns = outSchema.getColumns(); + List<Column> columns = outSchema.getRootColumns(); Tuple aTuple; for (TableDescriptorProto table: tables) { @@ -245,7 +245,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult private List<Tuple> getColumns(Schema outSchema) { List<ColumnProto> columnsList = masterContext.getCatalog().getAllColumns(); List<Tuple> tuples = new ArrayList<Tuple>(columnsList.size()); - List<Column> columns = outSchema.getColumns(); + List<Column> columns = outSchema.getRootColumns(); Tuple aTuple; int columnId = 1, prevtid = -1, tid = 0; @@ -293,7 +293,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult private List<Tuple> getIndexes(Schema outSchema) { List<IndexProto> indexList = masterContext.getCatalog().getAllIndexes(); List<Tuple> tuples = new ArrayList<Tuple>(indexList.size()); - List<Column> columns = outSchema.getColumns(); + List<Column> columns = outSchema.getRootColumns(); Tuple aTuple; for (IndexProto index: indexList) { @@ -332,7 +332,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult private List<Tuple> getAllTableOptions(Schema outSchema) { List<TableOptionProto> optionList = masterContext.getCatalog().getAllTableOptions(); List<Tuple> tuples = new ArrayList<Tuple>(optionList.size()); - List<Column> columns = outSchema.getColumns(); + List<Column> columns = outSchema.getRootColumns(); Tuple aTuple; for (TableOptionProto option: optionList) { @@ -359,7 +359,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult private List<Tuple> getAllTableStats(Schema outSchema) { List<TableStatsProto> statList = masterContext.getCatalog().getAllTableStats(); List<Tuple> tuples = new ArrayList<Tuple>(statList.size()); - List<Column> columns = outSchema.getColumns(); + List<Column> columns = outSchema.getRootColumns(); Tuple aTuple; for (TableStatsProto stat: statList) { @@ -386,7 +386,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult private List<Tuple> getAllPartitions(Schema outSchema) { List<TablePartitionProto> partitionList = masterContext.getCatalog().getAllPartitions(); List<Tuple> tuples = new ArrayList<Tuple>(partitionList.size()); - List<Column> columns = outSchema.getColumns(); + List<Column> columns = outSchema.getRootColumns(); Tuple aTuple; for (TablePartitionProto partition: partitionList) { @@ -417,7 +417,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult } private Tuple getQueryMasterTuple(Schema outSchema, Worker aWorker) { - List<Column> columns = outSchema.getColumns(); + List<Column> columns = outSchema.getRootColumns(); Tuple aTuple = new VTuple(outSchema.size()); WorkerResource aResource = aWorker.getResource(); @@ -463,7 +463,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult } private Tuple getWorkerTuple(Schema outSchema, Worker aWorker) { - List<Column> columns = outSchema.getColumns(); + List<Column> columns = outSchema.getRootColumns(); Tuple aTuple = new VTuple(outSchema.size()); WorkerResource aResource = aWorker.getResource(); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/resources/webapps/admin/catalogview.jsp ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/resources/webapps/admin/catalogview.jsp b/tajo-core/src/main/resources/webapps/admin/catalogview.jsp index 1ff81a6..e014379 100644 --- a/tajo-core/src/main/resources/webapps/admin/catalogview.jsp +++ b/tajo-core/src/main/resources/webapps/admin/catalogview.jsp @@ -30,7 +30,6 @@ <%@ page import="java.util.Collection" %> <%@ page import="java.util.List" %> <%@ page import="java.util.Map" %> -<%@ page import="org.apache.tajo.service.ServiceTracker" %> <% TajoMaster master = (TajoMaster) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object"); CatalogService catalog = master.getCatalog(); @@ -143,7 +142,7 @@ <div style='margin-top:5px'> <% if(tableDesc != null) { - List<Column> columns = tableDesc.getSchema().getColumns(); + List<Column> columns = tableDesc.getSchema().getRootColumns(); out.write("<table border='1' class='border_table'><tr><th>No</th><th>Column name</th><th>Type</th></tr>"); int columnIndex = 1; for(Column eachColumn: columns) { @@ -155,7 +154,7 @@ if (tableDesc.getPartitionMethod() != null) { PartitionMethodDesc partition = tableDesc.getPartitionMethod(); - List<Column> partitionColumns = partition.getExpressionSchema().getColumns(); + List<Column> partitionColumns = partition.getExpressionSchema().getRootColumns(); String partitionColumnStr = ""; String prefix = ""; for (Column eachColumn: partitionColumns) { http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java index 32b98dd..cf34c33 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java @@ -20,8 +20,7 @@ package org.apache.tajo.engine.function; import com.google.common.collect.Lists; import org.apache.tajo.catalog.FunctionDesc; -import org.apache.tajo.function.FunctionSignature; -import org.apache.tajo.util.TUtil; +import org.apache.tajo.util.StringUtils; import org.junit.Test; import java.io.IOException; @@ -38,7 +37,7 @@ public class TestFunctionLoader { public void testFindScalarFunctions() throws IOException { List<FunctionDesc> collections = Lists.newArrayList(FunctionLoader.findScalarFunctions()); Collections.sort(collections); - String functionList = TUtil.collectionToString(collections, "\n"); + String functionList = StringUtils.join(collections, "\n"); String result = getResultText(TestFunctionLoader.class, "testFindScalarFunctions.result"); assertEquals(result.trim(), functionList.trim()); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index af0aa6a..dfac53d 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -871,7 +871,7 @@ public class TestLogicalPlanner { assertEquals(NodeType.EXPRS, root.getChild().getType()); Schema out = root.getOutSchema(); - Iterator<Column> it = out.getColumns().iterator(); + Iterator<Column> it = out.getRootColumns().iterator(); Column col = it.next(); assertEquals("res1", col.getSimpleName()); col = it.next(); @@ -920,7 +920,7 @@ public class TestLogicalPlanner { testJsonSerDerObject(root); Schema finalSchema = root.getOutSchema(); - Iterator<Column> it = finalSchema.getColumns().iterator(); + Iterator<Column> it = finalSchema.getRootColumns().iterator(); Column col = it.next(); assertEquals("deptname", col.getSimpleName()); col = it.next(); @@ -931,7 +931,7 @@ public class TestLogicalPlanner { root = (LogicalRootNode) plan; finalSchema = root.getOutSchema(); - it = finalSchema.getColumns().iterator(); + it = finalSchema.getRootColumns().iterator(); col = it.next(); assertEquals("id", col.getSimpleName()); col = it.next(); @@ -948,7 +948,7 @@ public class TestLogicalPlanner { testJsonSerDerObject(root); Schema finalSchema = root.getOutSchema(); - Iterator<Column> it = finalSchema.getColumns().iterator(); + Iterator<Column> it = finalSchema.getRootColumns().iterator(); Column col = it.next(); assertEquals("id", col.getSimpleName()); col = it.next(); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java index f79f703..727b1a2 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java @@ -68,7 +68,7 @@ public class TestCTASQuery extends QueryTestCaseBase { assertTrue(desc.getSchema().contains("default.testctaswithouttabledefinition.col1")); PartitionMethodDesc partitionDesc = desc.getPartitionMethod(); assertEquals(partitionDesc.getPartitionType(), CatalogProtos.PartitionType.COLUMN); - assertEquals("key", partitionDesc.getExpressionSchema().getColumns().get(0).getSimpleName()); + assertEquals("key", partitionDesc.getExpressionSchema().getRootColumns().get(0).getSimpleName()); FileSystem fs = FileSystem.get(testBase.getTestingCluster().getConfiguration()); Path path = new Path(desc.getPath()); @@ -111,7 +111,7 @@ public class TestCTASQuery extends QueryTestCaseBase { assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); PartitionMethodDesc partitionDesc = desc.getPartitionMethod(); assertEquals(partitionDesc.getPartitionType(), CatalogProtos.PartitionType.COLUMN); - assertEquals("key", partitionDesc.getExpressionSchema().getColumns().get(0).getSimpleName()); + assertEquals("key", partitionDesc.getExpressionSchema().getRootColumns().get(0).getSimpleName()); FileSystem fs = FileSystem.get(cluster.getConfiguration()); Path path = new Path(desc.getPath()); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java index 48aea26..a1eceea 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java @@ -707,7 +707,7 @@ public class TestJoinBroadcast extends QueryTestCaseBase { "select a.o_orderdate, a.o_orderstatus, a.o_orderkey_mod, a.o_totalprice " + "from " + tableName + " a join "+ tableName + " b on a.o_orderkey = b.o_orderkey " + - "where a.o_orderdate = '1993-10-14' and a.o_orderstatus = 'F' and o_orderkey_mod = 1 " + + "where a.o_orderdate = '1993-10-14' and a.o_orderstatus = 'F' and a.o_orderkey_mod = 1 " + " order by a.o_orderkey" ); String resultSetData = resultSetToString(res); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java new file mode 100644 index 0000000..9f8a5fd --- /dev/null +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.query; + +import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.util.TUtil; +import org.junit.Test; + +import java.sql.ResultSet; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class TestSelectNestedRecord extends QueryTestCaseBase { + + @Test + public final void testSelect1() throws Exception { + List<String> tables = executeDDL("sample1_ddl.sql", "sample1", "sample1"); + assertEquals(TUtil.newList("sample1"), tables); + + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public final void testSelect2() throws Exception { + List<String> tables = executeDDL("tweets_ddl.sql", "tweets", "tweets"); + assertEquals(TUtil.newList("tweets"), tables); + + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public final void testNestedFieldAsGroupbyKey1() throws Exception { + List<String> tables = executeDDL("tweets_ddl.sql", "tweets", "tweets"); + assertEquals(TUtil.newList("tweets"), tables); + + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public final void testNestedFieldAsJoinKey1() throws Exception { + List<String> tables = executeDDL("tweets_ddl.sql", "tweets", "tweets"); + assertEquals(TUtil.newList("tweets"), tables); + + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java index 0dbb8e5..c8c24cd 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java @@ -262,7 +262,7 @@ public class TestTajoJdbc extends QueryTestCaseBase { TableDesc tableDesc = client.getTableDesc(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName)); assertNotNull(tableDesc); - List<Column> columns = tableDesc.getSchema().getColumns(); + List<Column> columns = tableDesc.getSchema().getRootColumns(); while (rs.next()) { assertEquals(tableName, rs.getString("TABLE_NAME")); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java b/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java index 6fc4ea1..1b23966 100644 --- a/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java +++ b/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java @@ -140,7 +140,7 @@ public class TestQueryResultResource extends QueryTestCaseBase { assertNotNull(response.getResultCode()); assertEquals(ResultCode.OK, response.getResultCode()); assertNotNull(response.getSchema()); - assertEquals(16, response.getSchema().getColumns().size()); + assertEquals(16, response.getSchema().getRootColumns().size()); assertNotNull(response.getResultset()); assertTrue(response.getResultset().getId() != 0); assertNotNull(response.getResultset().getLink()); @@ -174,7 +174,7 @@ public class TestQueryResultResource extends QueryTestCaseBase { assertNotNull(response.getResultCode()); assertEquals(ResultCode.OK, response.getResultCode()); assertNotNull(response.getSchema()); - assertEquals(16, response.getSchema().getColumns().size()); + assertEquals(16, response.getSchema().getRootColumns().size()); assertNotNull(response.getResultset()); assertTrue(response.getResultset().getId() != 0); assertNotNull(response.getResultset().getLink()); @@ -236,7 +236,7 @@ public class TestQueryResultResource extends QueryTestCaseBase { assertNotNull(response.getResultCode()); assertEquals(ResultCode.OK, response.getResultCode()); assertNotNull(response.getSchema()); - assertEquals(16, response.getSchema().getColumns().size()); + assertEquals(16, response.getSchema().getRootColumns().size()); assertNotNull(response.getResultset()); assertTrue(response.getResultset().getId() != 0); assertNotNull(response.getResultset().getLink()); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample1/table.json ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample1/table.json b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample1/table.json new file mode 100644 index 0000000..db3ad6c --- /dev/null +++ b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample1/table.json @@ -0,0 +1,3 @@ +{ "title" : "Hand of the King", "name" : { "first_name": "Eddard", "last_name": "Stark"}} +{ "title" : "Assassin", "name" : { "first_name": "Arya", "last_name": "Stark"}} +{ "title" : "Dancing Master", "name" : { "first_name": "Syrio", "last_name": "Forel"}} http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/tweets/sample1.json ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/tweets/sample1.json b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/tweets/sample1.json new file mode 100644 index 0000000..78a9071 --- /dev/null +++ b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/tweets/sample1.json @@ -0,0 +1,4 @@ +{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Mon Sep 24 03:35:21 +0000 2012","id_str":"250075927172759552","entities":{"urls":[],"hashtags":[{"text":"freebandnames","indices":[20,34]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"Aggressive Ponytail #freebandnames","metadata":{"iso_language_code":"en","result_type":"recent"},"retweet_count":1,"in_reply_to_status_id_str":null,"id":250075927172759552,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"DDEEF6","profile_sidebar_border_color":"C0DEED","profile_background_tile":false,"name":"Sean Cummings","profile_image_url":"http://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg","created_at":"Mon Apr 26 06:01:55 +0000 2010","location":"LA, CA","follow_request_sent":null,"profile_link_color":"0084B4","is_translator":false,"id_str":"137238150","entities":{"url":{"urls":[{"expanded_url":null,"url":"","i ndices":[0,0]}]},"description":{"urls":[]}},"default_profile":true,"contributors_enabled":false,"favourites_count":0,"url":null,"profile_image_url_https":"https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg","utc_offset":-28800,"id":137238150,"profile_use_background_image":true,"listed_count":2,"profile_text_color":"333333","lang":"en","followers_count":70,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme1/bg.png","profile_background_color":"C0DEED","verified":false,"geo_enabled":true,"time_zone":"Pacific Time (US & Canada)","description":"Born 330 Live 310","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/images/themes/theme1/bg.png","statuses_count":579,"friends_count":110,"following":null,"show_all_inline_media":false,"screen_name":"sean_cummings"},"in_reply_to_screen_name":null,"source":";<a>Twitter for Mac<\/a>","in_reply_to_status_id":null} +{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Fri Sep 21 23:40:54 +0000 2012","id_str":"249292149810667520","entities":{"urls":[],"hashtags":[{"text":"FreeBandNames","indices":[20,34]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"Thee Namaste Nerdz. #FreeBandNames","metadata":{"iso_language_code":"pl","result_type":"recent"},"retweet_count":2,"in_reply_to_status_id_str":null,"id":249292149810667520,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"DDFFCC","profile_sidebar_border_color":"BDDCAD","profile_background_tile":true,"name":"Chaz Martenstein","profile_image_url":"http://a0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg","created_at":"Tue Apr 07 19:05:07 +0000 2009","location":"Durham, NC","follow_request_sent":null,"profile_link_color":"0084B4","is_translator":false,"id_str":"29516238","entities":{"url":{"urls":[{"expanded_url":null,"url":"http://bu llcityrecords.com/wnng/","indices":[0,32]}]},"description":{"urls":[]}},"default_profile":false,"contributors_enabled":false,"favourites_count":8,"url":"http://bullcityrecords.com/wnng/","profile_image_url_https":"https://si0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg","utc_offset":-18000,"id":29516238,"profile_use_background_image":true,"listed_count":118,"profile_text_color":"333333","lang":"en","followers_count":2052,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/profile_background_images/9423277/background_tile.bmp","profile_background_color":"9AE4E8","verified":false,"geo_enabled":false,"time_zone":"Eastern Time (US & Canada)","description":"You will come to Durham, North Carolina. I will sell you some records then, here in Durham, North Carolina. Fun will happen.","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/profile_background_images/9423277/background_tile.bmp","statuses_c ount":7579,"friends_count":348,"following":null,"show_all_inline_media":true,"screen_name":"bullcityrecords"},"in_reply_to_screen_name":null,"source":"web","in_reply_to_status_id":null} +{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Fri Sep 21 23:30:20 +0000 2012","id_str":"249289491129438208","entities":{"urls":[],"hashtags":[{"text":"freebandnames","indices":[29,43]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"Mexican Heaven, Mexican Hell #freebandnames","metadata":{"iso_language_code":"en","result_type":"recent"},"retweet_count":3,"in_reply_to_status_id_str":null,"id":249289491129438208,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"99CC33","profile_sidebar_border_color":"829D5E","profile_background_tile":false,"name":"Thomas John Wakeman","profile_image_url":"http://a0.twimg.com/profile_images/2219333930/Froggystyle_normal.png","created_at":"Tue Sep 01 21:21:35 +0000 2009","location":"Kingston New York","follow_request_sent":null,"profile_link_color":"D02B55","is_translator":false,"id_str":"70789458","entities":{"url":{"urls":[{"expanded_url":n ull,"url":"","indices":[0,0]}]},"description":{"urls":[]}},"default_profile":false,"contributors_enabled":false,"favourites_count":19,"url":null,"profile_image_url_https":"https://si0.twimg.com/profile_images/2219333930/Froggystyle_normal.png","utc_offset":-18000,"id":70789458,"profile_use_background_image":true,"listed_count":1,"profile_text_color":"3E4415","lang":"en","followers_count":63,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme5/bg.gif","profile_background_color":"352726","verified":false,"geo_enabled":false,"time_zone":"Eastern Time (US & Canada)","description":"Science Fiction Writer, sort of. Likes Superheroes, Mole People, Alt. Timelines.","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/images/themes/theme5/bg.gif","statuses_count":1048,"friends_count":63,"following":null,"show_all_inline_media":false,"screen_name":"MonkiesFist"},"in_reply_to_screen_name":null,"sour ce":"web","in_reply_to_status_id":null} +{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Fri Sep 21 22:51:18 +0000 2012","id_str":"249279667666817024","entities":{"urls":[],"hashtags":[{"text":"freebandnames","indices":[20,34]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"The Foolish Mortals #freebandnames","metadata":{"iso_language_code":"en","result_type":"recent"},"retweet_count":4,"in_reply_to_status_id_str":null,"id":249279667666817024,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"BFAC83","profile_sidebar_border_color":"615A44","profile_background_tile":true,"name":"Marty Elmer","profile_image_url":"http://a0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png","created_at":"Mon May 04 00:05:00 +0000 2009","location":"Wisconsin, USA","follow_request_sent":null,"profile_link_color":"3B2A26","is_translator":false,"id_str":"37539828","entities":{"url":{"urls":[{"expanded_url":null,"url":"ht tp://www.omnitarian.me","indices":[0,24]}]},"description":{"urls":[]}},"default_profile":false,"contributors_enabled":false,"favourites_count":647,"url":"http://www.omnitarian.me","profile_image_url_https":"https://si0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png","utc_offset":-21600,"id":37539828,"profile_use_background_image":true,"listed_count":52,"profile_text_color":"000000","lang":"en","followers_count":608,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/profile_background_images/106455659/rect6056-9.png","profile_background_color":"EEE3C4","verified":false,"geo_enabled":false,"time_zone":"Central Time (US & Canada)","description":"Cartoonist, Illustrator, and T-Shirt connoisseur","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/profile_background_images/106455659/rect6056-9.png","statuses_count":3575,"friends_count":249,"following":null,"show_all_inline_media":true,"scree n_name":"Omnitarian"},"in_reply_to_screen_name":null,"source":"<a>Twitter for iPhone<\/a>","in_reply_to_status_id":null} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample1_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample1_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample1_ddl.sql new file mode 100644 index 0000000..9ba5f8c --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample1_ddl.sql @@ -0,0 +1,7 @@ +CREATE EXTERNAL TABLE ${0} ( + title TEXT, + name RECORD ( + first_name TEXT, + last_name TEXT + ) +) USING JSON LOCATION ${table.path}; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql new file mode 100644 index 0000000..9537c3e --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql @@ -0,0 +1,19 @@ +CREATE EXTERNAL TABLE ${0} ( + glossary RECORD ( + title TEXT, + "GlossDiv" RECORD ( + "GlossEntry" RECORD ( + "ID" TEXT, + "SortAs" TEXT, + "GlossTerm" TEXT, + "Acronym" TEXT, + "Abbrev" TEXT, + "GlossDef" RECORD ( + para TEXT + ), + + "GlossSee" TEXT + ) + ) + ) +) USING JSON LOCATION ${path}; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.sql new file mode 100644 index 0000000..057ba05 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.sql @@ -0,0 +1,7 @@ +SELECT + user.name, + sum(retweet_count) as total_retweet +FROM + tweets +GROUP BY + user.name; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsJoinKey1.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsJoinKey1.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsJoinKey1.sql new file mode 100644 index 0000000..336840e --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsJoinKey1.sql @@ -0,0 +1,7 @@ +SELECT + t1.user.id, + t1.user.name, + t2.user.id, + t2.user.name +FROM + tweets t1 join tweets t2 ON t1.user.id = t2.user.id \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect1.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect1.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect1.sql new file mode 100644 index 0000000..b099e77 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect1.sql @@ -0,0 +1 @@ +SELECT title, (name.first_name || ' ' || name.last_name) as full_name FROM sample1; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect2.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect2.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect2.sql new file mode 100644 index 0000000..9993a4e --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect2.sql @@ -0,0 +1,61 @@ +SELECT + coordinates, + favorited, + truncated, + created_at, + id_str, + in_reply_to_user_id_str, + contributors, + "text", + metadata.iso_language_code, + metadata.result_type, + retweet_count, + in_reply_to_status_id_str, + id, + geo, + retweeted, + in_reply_to_user_id, + place, + user.profile_sidebar_fill_color, + user.profile_sidebar_border_color, + user.profile_background_tile, + user.name, + user.profile_image_url, + user.created_at, + user.location, + user.follow_request_sent, + user.profile_link_color, + user.is_translator, + user.id_str, + user.default_profile, + user.contributors_enabled, + user.favourites_count, + user.url, + user.profile_image_url_https, + user.utc_offset, + user.id, + user.profile_use_background_image, + user.listed_count, + user.profile_text_color, + user.lang, + user.followers_count, + user.protected, + user.notifications, + user.profile_background_image_url_https, + user.profile_background_color, + user.verified, + user.geo_enabled, + user.time_zone, + user.description, + user.default_profile_image, + user.profile_background_image_url, + user.statuses_count, + user.friends_count, + user.following, + user.show_all_inline_media, + user.screen_name, + in_reply_to_screen_name, + source, + in_reply_to_status_id +FROM + tweets; \ No newline at end of file
![http://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg","created_at":"Mon](https://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg","created_at":"Mon){kind=link}
![https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg","utc_offset":-28800,"id":137238150,"profile_use_background_image":true,"listed_count":2,"profile_text_color":"333333","lang":"en","followers_count":70,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme1/bg.png","profile_background_color":"C0DEED","verified":false,"geo_enabled":true,"time_zone":"Pacific](https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg","utc_offset":-28800,"id":137238150,"profile_use_background_image":true,"listed_count":2,"profile_text_color":"333333","lang":"en","followers_count":70,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme1/bg.png","profile_background_color":"C0DEED","verified":false,"geo_enabled":true,"time_zone":"Pacific){kind=link}
![http://a0.twimg.com/images/themes/theme1/bg.png","statuses_count":579,"friends_count":110,"following":null,"show_all_inline_media":false,"screen_name":"sean_cummings"},"in_reply_to_screen_name":null,"source":"](https://a0.twimg.com/images/themes/theme1/bg.png","statuses_count":579,"friends_count":110,"following":null,"show_all_inline_media":false,"screen_name":"sean_cummings"},"in_reply_to_screen_name":null,"source":"){kind=link}
![http://a0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg","created_at":"Tue](https://a0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg","created_at":"Tue){kind=link}
![http://bullcityrecords.com/wnng/","profile_image_url_https":"https://si0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg","utc_offset":-18000,"id":29516238,"profile_use_background_image":true,"listed_count":118,"profile_text_color":"333333","lang":"en","followers_count":2052,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/profile_background_images/9423277/background_tile.bmp","profile_background_color":"9AE4E8","verified":false,"geo_enabled":false,"time_zone":"Eastern](http://bullcityrecords.com/wnng/","profile_image_url_https":"https://si0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg","utc_offset":-18000,"id":29516238,"profile_use_background_image":true,"listed_count":118,"profile_text_color":"333333","lang":"en","followers_count":2052,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/profile_background_images/9423277/background_tile.bmp","profile_background_color":"9AE4E8","verified":false,"geo_enabled":false,"time_zone":"Eastern){kind=link}
![http://a0.twimg.com/profile_background_images/9423277/background_tile.bmp","statuses_c](https://a0.twimg.com/profile_background_images/9423277/background_tile.bmp","statuses_c){kind=link}
![http://a0.twimg.com/profile_images/2219333930/Froggystyle_normal.png","created_at":"Tue](https://a0.twimg.com/profile_images/2219333930/Froggystyle_normal.png","created_at":"Tue){kind=link}
![https://si0.twimg.com/profile_images/2219333930/Froggystyle_normal.png","utc_offset":-18000,"id":70789458,"profile_use_background_image":true,"listed_count":1,"profile_text_color":"3E4415","lang":"en","followers_count":63,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme5/bg.gif","profile_background_color":"352726","verified":false,"geo_enabled":false,"time_zone":"Eastern](https://si0.twimg.com/profile_images/2219333930/Froggystyle_normal.png","utc_offset":-18000,"id":70789458,"profile_use_background_image":true,"listed_count":1,"profile_text_color":"3E4415","lang":"en","followers_count":63,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme5/bg.gif","profile_background_color":"352726","verified":false,"geo_enabled":false,"time_zone":"Eastern){kind=link}
![http://a0.twimg.com/images/themes/theme5/bg.gif","statuses_count":1048,"friends_count":63,"following":null,"show_all_inline_media":false,"screen_name":"MonkiesFist"},"in_reply_to_screen_name":null,"sour](https://a0.twimg.com/images/themes/theme5/bg.gif","statuses_count":1048,"friends_count":63,"following":null,"show_all_inline_media":false,"screen_name":"MonkiesFist"},"in_reply_to_screen_name":null,"sour){kind=link}
![http://a0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png","created_at":"Mon](https://a0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png","created_at":"Mon){kind=link}
![http://www.omnitarian.me","profile_image_url_https":"https://si0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png","utc_offset":-21600,"id":37539828,"profile_use_background_image":true,"listed_count":52,"profile_text_color":"000000","lang":"en","followers_count":608,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/profile_background_images/106455659/rect6056-9.png","profile_background_color":"EEE3C4","verified":false,"geo_enabled":false,"time_zone":"Central](http://www.omnitarian.me","profile_image_url_https":"https://si0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png","utc_offset":-21600,"id":37539828,"profile_use_background_image":true,"listed_count":52,"profile_text_color":"000000","lang":"en","followers_count":608,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/profile_background_images/106455659/rect6056-9.png","profile_background_color":"EEE3C4","verified":false,"geo_enabled":false,"time_zone":"Central){kind=link}
![http://a0.twimg.com/profile_background_images/106455659/rect6056-9.png","statuses_count":3575,"friends_count":249,"following":null,"show_all_inline_media":true,"scree](https://a0.twimg.com/profile_background_images/106455659/rect6056-9.png","statuses_count":3575,"friends_count":249,"following":null,"show_all_inline_media":true,"scree){kind=link}