http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/tweets_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/tweets_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/tweets_ddl.sql new file mode 100644 index 0000000..e750095 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/tweets_ddl.sql @@ -0,0 +1,74 @@ +CREATE EXTERNAL TABLE IF NOT EXISTS ${0} ( + coordinates TEXT, + favorited BOOL, + truncated BOOL, + created_at TIMESTAMP, + id_str TEXT, + /*entrities RECORD ( -- when we support array, we should remove this comment. + urls ARRAY<TEXT> + )*/ + in_reply_to_user_id_str TEXT, + contributors TEXT, + text TEXT, + metadata RECORD ( + iso_language_code TEXT, + result_type TEXT + ), + retweet_count INTEGER, + in_reply_to_status_id_str TEXT, + id TEXT, + geo TEXT, + retweeted BOOL, + in_reply_to_user_id TEXT, + place TEXT, + user RECORD ( + profile_sidebar_fill_color TEXT, + profile_sidebar_border_color TEXT, + profile_background_tile TEXT, + name TEXT, + profile_image_url TEXT, + created_at TIMESTAMP, + location TEXT, + follow_request_sent TEXT, + profile_link_color TEXT, + is_translator BOOL, + id_str TEXT, + /* -- when we support array, we should fill the following comments. + entities RECORD ( + url RECORD ( + ), + description RECORD ( + ) + ), */ + default_profile BOOL, + contributors_enabled BOOL, + favourites_count INTEGER, + url TEXT, + profile_image_url_https TEXT, + utc_offset INTEGER, + id BIGINT, + profile_use_background_image BOOL, + listed_count INTEGER, + profile_text_color TEXT, + lang TEXT, + followers_count INTEGER, + protected BOOL, + notifications TEXT, + profile_background_image_url_https TEXT, + profile_background_color TEXT, + verified TEXT, + geo_enabled TEXT, + time_zone TEXT, + description TEXT, + default_profile_image TEXT, + profile_background_image_url TEXT, + statuses_count INTEGER, + friends_count INTEGER, + following TEXT, + show_all_inline_media BOOL, + screen_name TEXT + ), + in_reply_to_screen_name TEXT, + source TEXT, + in_reply_to_status_id TEXT +) USING JSON LOCATION ${table.path}; \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql index 1760c99..e707a8c 100644 --- a/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql @@ -1 +1 @@ -(select n1.n_nationkey, n2.n_name from nation n1 join nation n2 on n1.n_nationkey = n2.n_nationkey where n_nationkey = 1); +(select n1.n_nationkey, n2.n_name from nation n1 join nation n2 on n1.n_nationkey = n2.n_nationkey where n1.n_nationkey = 1); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.result new file mode 100644 index 0000000..debf06e --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.result @@ -0,0 +1,6 @@ +user/name,total_retweet +------------------------------- +Chaz Martenstein,2 +Thomas John Wakeman,3 +Sean Cummings,1 +Marty Elmer,4 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsJoinKey1.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsJoinKey1.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsJoinKey1.result new file mode 100644 index 0000000..1c57dc2 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsJoinKey1.result @@ -0,0 +1,6 @@ +user/id,user/name,user/id,user/name +------------------------------- +137238150,Sean Cummings,137238150,Sean Cummings +29516238,Chaz Martenstein,29516238,Chaz Martenstein +70789458,Thomas John Wakeman,70789458,Thomas John Wakeman +37539828,Marty Elmer,37539828,Marty Elmer \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect1.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect1.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect1.result new file mode 100644 index 0000000..f96fcc5 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect1.result @@ -0,0 +1,5 @@ +title,full_name +------------------------------- +Hand of the King,Eddard Stark +Assassin,Arya Stark +Dancing Master,Syrio Forel \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect2.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect2.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect2.result new file mode 100644 index 0000000..90eadbd --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect2.result @@ -0,0 +1,6 @@ +coordinates,favorited,truncated,created_at,id_str,in_reply_to_user_id_str,contributors,text,metadata/iso_language_code,metadata/result_type,retweet_count,in_reply_to_status_id_str,id,geo,retweeted,in_reply_to_user_id,place,user/profile_sidebar_fill_color,user/profile_sidebar_border_color,user/profile_background_tile,user/name,user/profile_image_url,user/created_at,user/location,user/follow_request_sent,user/profile_link_color,user/is_translator,user/id_str,user/default_profile,user/contributors_enabled,user/favourites_count,user/url,user/profile_image_url_https,user/utc_offset,user/id,user/profile_use_background_image,user/listed_count,user/profile_text_color,user/lang,user/followers_count,user/protected,user/notifications,user/profile_background_image_url_https,user/profile_background_color,user/verified,user/geo_enabled,user/time_zone,user/description,user/default_profile_image,user/profile_background_image_url,user/statuses_count,user/friends_count,user/following,user/show_all_in line_media,user/screen_name,in_reply_to_screen_name,source,in_reply_to_status_id +------------------------------- +null,false,false,2012-09-24 03:35:21,250075927172759552,null,null,Aggressive Ponytail #freebandnames,en,recent,1,null,250075927172759552,null,false,null,null,DDEEF6,C0DEED,false,Sean Cummings,http://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg,2010-04-26 06:01:55,LA, CA,null,0084B4,false,137238150,true,false,0,null,https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg,-28800,137238150,true,2,333333,en,70,false,null,https://si0.twimg.com/images/themes/theme1/bg.png,C0DEED,false,true,Pacific Time (US & Canada),Born 330 Live 310,false,http://a0.twimg.com/images/themes/theme1/bg.png,579,110,null,false,sean_cummings,null,<a>Twitter for Mac</a>,null +null,false,false,2012-09-21 23:40:54,249292149810667520,null,null,Thee Namaste Nerdz. #FreeBandNames,pl,recent,2,null,249292149810667520,null,false,null,null,DDFFCC,BDDCAD,true,Chaz Martenstein,http://a0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg,2009-04-07 19:05:07,Durham, NC,null,0084B4,false,29516238,false,false,8,http://bullcityrecords.com/wnng/,https://si0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg,-18000,29516238,true,118,333333,en,2052,false,null,https://si0.twimg.com/profile_background_images/9423277/background_tile.bmp,9AE4E8,false,false,Eastern Time (US & Canada),You will come to Durham, North Carolina. I will sell you some records then, here in Durham, North Carolina. Fun will happen.,false,http://a0.twimg.com/profile_background_images/9423277/background_tile.bmp,7579,348,null,true,bullcityrecords,null,web,null +null,false,false,2012-09-21 23:30:20,249289491129438208,null,null,Mexican Heaven, Mexican Hell #freebandnames,en,recent,3,null,249289491129438208,null,false,null,null,99CC33,829D5E,false,Thomas John Wakeman,http://a0.twimg.com/profile_images/2219333930/Froggystyle_normal.png,2009-09-01 21:21:35,Kingston New York,null,D02B55,false,70789458,false,false,19,null,https://si0.twimg.com/profile_images/2219333930/Froggystyle_normal.png,-18000,70789458,true,1,3E4415,en,63,false,null,https://si0.twimg.com/images/themes/theme5/bg.gif,352726,false,false,Eastern Time (US & Canada),Science Fiction Writer, sort of. Likes Superheroes, Mole People, Alt. Timelines.,false,http://a0.twimg.com/images/themes/theme5/bg.gif,1048,63,null,false,MonkiesFist,null,web,null +null,false,false,2012-09-21 22:51:18,249279667666817024,null,null,The Foolish Mortals #freebandnames,en,recent,4,null,249279667666817024,null,false,null,null,BFAC83,615A44,true,Marty Elmer,http://a0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png,2009-05-04 00:05:00,Wisconsin, USA,null,3B2A26,false,37539828,false,false,647,http://www.omnitarian.me,https://si0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png,-21600,37539828,true,52,000000,en,608,false,null,https://si0.twimg.com/profile_background_images/106455659/rect6056-9.png,EEE3C4,false,false,Central Time (US & Canada),Cartoonist, Illustrator, and T-Shirt connoisseur,false,http://a0.twimg.com/profile_background_images/106455659/rect6056-9.png,3575,249,null,true,Omnitarian,null,<a>Twitter for iPhone</a>,null \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java ---------------------------------------------------------------------- diff --git a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java index 2368082..d4ef55e 100644 --- a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java +++ b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java @@ -513,7 +513,7 @@ public class TajoDatabaseMetaData implements DatabaseMetaData { CatalogUtil.buildFQName(databaseName, table)); int pos = 0; - for (Column column: tableDesc.getLogicalSchema().getColumns()) { + for (Column column: tableDesc.getLogicalSchema().getRootColumns()) { if (column.getSimpleName().matches(regcolumnNamePattern)) { MetaDataTuple tuple = new MetaDataTuple(22); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java index 6262995..fbad76e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java @@ -114,7 +114,7 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanner.P Schema schema = relationOp.getLogicalSchema(); Column[] resolvedColumns = new Column[schema.size()]; - return schema.getColumns().toArray(resolvedColumns); + return schema.getRootColumns().toArray(resolvedColumns); } else { // if a column reference is not qualified // columns of every relation should be resolved. Iterator<RelationNode> iterator = block.getRelations().iterator(); @@ -124,7 +124,7 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanner.P while (iterator.hasNext()) { relationOp = iterator.next(); schema = relationOp.getLogicalSchema(); - resolvedColumns.addAll(schema.getColumns()); + resolvedColumns.addAll(schema.getRootColumns()); } if (resolvedColumns.size() == 0) { http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index e0b4f7e..21270e9 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -52,6 +52,7 @@ import org.apache.tajo.catalog.SchemaUtil; import org.apache.tajo.plan.verifier.VerifyException; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.Pair; +import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; import java.util.*; @@ -527,13 +528,24 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex } else if (projectable instanceof RelationNode) { RelationNode relationNode = (RelationNode) projectable; + prohibitNestedRecordProjection((Projectable) relationNode); verifyIfTargetsCanBeEvaluated(relationNode.getLogicalSchema(), (Projectable) relationNode); } else { + prohibitNestedRecordProjection(projectable); verifyIfTargetsCanBeEvaluated(projectable.getInSchema(), projectable); } } + public static void prohibitNestedRecordProjection(Projectable projectable) + throws PlanningException { + for (Target t : projectable.getTargets()) { + if (t.getEvalTree().getValueType().getType() == TajoDataTypes.Type.RECORD) { + throw new PlanningException("Projecting RECORD fields is not supported yet: " + t); + } + } + } + public static void verifyIfEvalNodesCanBeEvaluated(Projectable projectable, EvalNode[] evalNodes) throws PlanningException { for (EvalNode e : evalNodes) { @@ -649,7 +661,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex if (block.namedExprsMgr.isEvaluated(sortKeyRefNames[i])) { column = block.namedExprsMgr.getTarget(sortKeyRefNames[i]).getNamedColumn(); } else { - throw new IllegalStateException("Unexpected State: " + TUtil.arrayToString(sortSpecs)); + throw new IllegalStateException("Unexpected State: " + StringUtils.join(sortSpecs)); } annotatedSortSpecs[i] = new SortSpec(column, sortSpecs[i].isAscending(), sortSpecs[i].isNullFirst()); } @@ -866,7 +878,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex } else if (block.namedExprsMgr.isEvaluated(refName)) { column = block.namedExprsMgr.getTarget(refName).getNamedColumn(); } else { - throw new IllegalStateException("Unexpected State: " + TUtil.arrayToString(sortSpecs)); + throw new IllegalStateException("Unexpected State: " + StringUtils.join(sortSpecs)); } annotatedSortSpecs.add(new SortSpec(column, sortSpecs[i].isAscending(), sortSpecs[i].isNullFirst())); } @@ -1175,13 +1187,13 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex Schema joinSchema = new Schema(); Schema commons = SchemaUtil.getNaturalJoinColumns(left.getOutSchema(), right.getOutSchema()); joinSchema.addColumns(commons); - for (Column c : left.getOutSchema().getColumns()) { + for (Column c : left.getOutSchema().getRootColumns()) { if (!joinSchema.contains(c.getQualifiedName())) { joinSchema.addColumn(c); } } - for (Column c : right.getOutSchema().getColumns()) { + for (Column c : right.getOutSchema().getRootColumns()) { if (!joinSchema.contains(c.getQualifiedName())) { joinSchema.addColumn(c); } @@ -1199,7 +1211,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex Column leftJoinKey; Column rightJoinKey; - for (Column common : commons.getColumns()) { + for (Column common : commons.getRootColumns()) { leftJoinKey = leftSchema.getColumn(common.getQualifiedName()); rightJoinKey = rightSchema.getColumn(common.getQualifiedName()); equiQual = new BinaryEval(EvalType.EQUAL, @@ -1312,7 +1324,15 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex private static LinkedHashSet<Target> createFieldTargetsFromRelation(QueryBlock block, RelationNode relationNode, Set<String> newlyEvaluatedRefNames) { LinkedHashSet<Target> targets = Sets.newLinkedHashSet(); - for (Column column : relationNode.getLogicalSchema().getColumns()) { + for (Column column : relationNode.getLogicalSchema().getAllColumns()) { + + // TODO - Currently, EvalNode has DataType as a return type. So, RECORD cannot be used for any target. + // The following line is a kind of hack, preventing RECORD to be used for target in the logical planning phase. + // This problem should be resolved after TAJO-1402. + if (column.getTypeDesc().getDataType().getType() == TajoDataTypes.Type.RECORD) { + continue; + } + String aliasName = block.namedExprsMgr.checkAndGetIfAliasedColumn(column.getQualifiedName()); if (aliasName != null) { targets.add(new Target(new FieldEval(column), aliasName)); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java index 153a150..eddb022 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java @@ -26,6 +26,7 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.storage.Tuple; +import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; import static org.apache.tajo.common.TajoDataTypes.DataType; @@ -88,7 +89,7 @@ public class RowConstantEval extends EvalNode { } public String toString() { - return TUtil.arrayToString(values); + return StringUtils.join(values); } public void preOrder(EvalNodeVisitor visitor) { http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java index 0b60d14..a39d303 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java @@ -26,6 +26,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.plan.logical.WindowSpec; import org.apache.tajo.storage.Tuple; +import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; import java.util.Arrays; @@ -120,7 +121,7 @@ public class WindowFunctionEval extends AggregationFunctionCallEval implements C sb.append(funcDesc.getFunctionName()).append("(").append(isDistinct() ? " distinct" : "").append(sb) .append(")"); if (hasSortSpecs()) { - sb.append("ORDER BY ").append(TUtil.arrayToString(sortSpecs)); + sb.append("ORDER BY ").append(StringUtils.join(sortSpecs)); } return sb.toString(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java index 878553f..079fc3e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java @@ -478,7 +478,7 @@ public class PythonScriptEngine extends TajoScriptEngine { try { result = outputHandler.getNext().get(0); } catch (Exception e) { - throw new RuntimeException("Problem getting output", e); + throw new RuntimeException("Problem getting output: " + e.getMessage(), e); } return result; @@ -512,7 +512,7 @@ public class PythonScriptEngine extends TajoScriptEngine { try { outputHandler.getNext(); } catch (Exception e) { - throw new RuntimeException("Problem getting output", e); + throw new RuntimeException("Problem getting output: " + e.getMessage(), e); } } @@ -532,7 +532,7 @@ public class PythonScriptEngine extends TajoScriptEngine { try { outputHandler.getNext(); } catch (Exception e) { - throw new RuntimeException("Problem getting output", e); + throw new RuntimeException("Problem getting output: " + e.getMessage(), e); } } @@ -552,7 +552,7 @@ public class PythonScriptEngine extends TajoScriptEngine { try { outputHandler.getNext(functionContext); } catch (Exception e) { - throw new RuntimeException("Problem getting output", e); + throw new RuntimeException("Problem getting output: " + e.getMessage(), e); } } @@ -573,7 +573,7 @@ public class PythonScriptEngine extends TajoScriptEngine { try { return outputHandler.getPartialResultString(); } catch (Exception e) { - throw new RuntimeException("Problem getting output", e); + throw new RuntimeException("Problem getting output: " + e.getMessage(), e); } } @@ -595,7 +595,7 @@ public class PythonScriptEngine extends TajoScriptEngine { try { result = outputHandler.getNext().get(0); } catch (Exception e) { - throw new RuntimeException("Problem getting output", e); + throw new RuntimeException("Problem getting output: " + e.getMessage(), e); } return result; http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java index e4d6122..862cb8a 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java @@ -26,7 +26,7 @@ import org.apache.tajo.plan.PlanningException; import org.apache.tajo.catalog.SchemaUtil; import org.apache.tajo.plan.expr.AlgebraicUtil; import org.apache.tajo.plan.logical.*; -import org.apache.tajo.util.TUtil; +import org.apache.tajo.util.StringUtils; import java.util.*; @@ -59,7 +59,7 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm { for (LogicalNode relation : remainRelations) { Collection <String> relationStrings = PlannerUtil.getRelationLineageWithinQueryBlock(plan, relation); List<JoinEdge> joinEdges = new ArrayList<JoinEdge>(); - String relationCollection = TUtil.collectionToString(relationStrings, ","); + String relationCollection = StringUtils.join(relationStrings, ","); List<JoinEdge> joinEdgesForGiven = joinGraph.getIncomingEdges(relationCollection); if (joinEdgesForGiven != null) { joinEdges.addAll(joinEdgesForGiven); @@ -236,7 +236,7 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm { // If outer is outer join, make edge key using all relation names in outer. SortedSet<String> relationNames = new TreeSet<String>(PlannerUtil.getRelationLineageWithinQueryBlock(plan, outer)); - String outerEdgeKey = TUtil.collectionToString(relationNames, ", "); + String outerEdgeKey = StringUtils.join(relationNames, ", "); for (String innerName : PlannerUtil.getRelationLineageWithinQueryBlock(plan, inner)) { if (graph.hasEdge(outerEdgeKey, innerName)) { JoinEdge existJoinEdge = graph.getEdge(outerEdgeKey, innerName); @@ -257,7 +257,7 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm { relationNames = new TreeSet<String>(PlannerUtil.getRelationLineageWithinQueryBlock(plan, inner)); - outerEdgeKey = TUtil.collectionToString(relationNames, ", "); + outerEdgeKey = StringUtils.join(relationNames, ", "); for (String outerName : PlannerUtil.getRelationLineageWithinQueryBlock(plan, outer)) { if (graph.hasEdge(outerEdgeKey, outerName)) { JoinEdge existJoinEdge = graph.getEdge(outerEdgeKey, outerName); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java index ce06748..fb4fae1 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java @@ -22,7 +22,7 @@ import com.google.common.collect.Sets; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.LogicalNode; -import org.apache.tajo.util.TUtil; +import org.apache.tajo.util.StringUtils; import java.util.Collections; import java.util.Set; @@ -70,6 +70,6 @@ public class JoinEdge { } public String toString() { - return leftRelation + " " + joinType + " " + rightRelation + " ON " + TUtil.collectionToString(joinQual, ", "); + return leftRelation + " " + joinType + " " + rightRelation + " ON " + StringUtils.join(joinQual, ", "); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java index 72e9b1d..9ae5245 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java @@ -22,6 +22,7 @@ import com.google.common.collect.Sets; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; +import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.graph.SimpleUndirectedGraph; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.NamedExprsManager; @@ -33,7 +34,6 @@ import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.expr.EvalTreeUtil; import org.apache.tajo.plan.logical.JoinNode; import org.apache.tajo.plan.logical.RelationNode; -import org.apache.tajo.util.TUtil; import java.util.*; @@ -111,8 +111,8 @@ public class JoinGraph extends SimpleUndirectedGraph<String, JoinEdge> { new TreeSet<String>(PlannerUtil.getRelationLineageWithinQueryBlock(plan, joinNode.getRightChild())); addEdge( - TUtil.collectionToString(leftNodeRelationName, ", "), - TUtil.collectionToString(rightNodeRelationName, ", "), + StringUtils.join(leftNodeRelationName, ", "), + StringUtils.join(rightNodeRelationName, ", "), edge); Set<EvalNode> allInOneCnf = new HashSet<EvalNode>(); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java index 61ce789..fb19f10 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java @@ -19,12 +19,12 @@ package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; - import org.apache.tajo.catalog.Column; import org.apache.tajo.plan.PlanString; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.Target; import org.apache.tajo.plan.expr.AggregationFunctionCallEval; +import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; import java.util.ArrayList; @@ -150,7 +150,7 @@ public class DistinctGroupbyNode extends UnaryNode implements Projectable, Clone public String toString() { StringBuilder sb = new StringBuilder("Distinct GroupBy ("); if (groupingColumns != null && groupingColumns.length > 0) { - sb.append("grouping set=").append(TUtil.arrayToString(groupingColumns)); + sb.append("grouping set=").append(StringUtils.join(groupingColumns)); sb.append(", "); } for (GroupbyNode eachNode: subGroupbyPlan) { http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java index 3aaf5d0..0632f68 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java @@ -28,6 +28,7 @@ import com.google.gson.annotations.Expose; import org.apache.tajo.plan.PlanString; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.Target; +import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; public class EvalExprNode extends LogicalNode implements Projectable { @@ -69,7 +70,7 @@ public class EvalExprNode extends LogicalNode implements Projectable { @Override public String toString() { - return "EvalExprNode (" + TUtil.arrayToString(exprs) + ")"; + return "EvalExprNode (" + StringUtils.join(exprs) + ")"; } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java index 730f705..23a9154 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java @@ -18,18 +18,18 @@ package org.apache.tajo.plan.logical; -import java.util.Arrays; - import com.google.common.base.Preconditions; import com.google.gson.annotations.Expose; - import org.apache.tajo.catalog.Column; import org.apache.tajo.plan.PlanString; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.Target; import org.apache.tajo.plan.expr.AggregationFunctionCallEval; +import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; +import java.util.Arrays; + public class GroupbyNode extends UnaryNode implements Projectable, Cloneable { /** Grouping key sets */ @Expose private Column [] groupingKeys = PlannerUtil.EMPTY_COLUMNS; @@ -110,11 +110,11 @@ public class GroupbyNode extends UnaryNode implements Projectable, Cloneable { public String toString() { StringBuilder sb = new StringBuilder("GroupBy ("); if (groupingKeys != null && groupingKeys.length > 0) { - sb.append("grouping set=").append(TUtil.arrayToString(groupingKeys)); + sb.append("grouping set=").append(StringUtils.join(groupingKeys)); sb.append(", "); } if (hasAggFunctions()) { - sb.append("funcs=").append(TUtil.arrayToString(aggrFunctions)); + sb.append("funcs=").append(StringUtils.join(aggrFunctions)); } sb.append(")"); return sb.toString(); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java index 3c9d497..ef6734e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java @@ -25,6 +25,7 @@ import com.google.gson.annotations.Expose; import org.apache.tajo.plan.PlanString; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.Target; +import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; public class ProjectionNode extends UnaryNode implements Projectable { @@ -70,7 +71,7 @@ public class ProjectionNode extends UnaryNode implements Projectable { public String toString() { StringBuilder sb = new StringBuilder("Projection (distinct=").append(distinct); if (targets != null) { - sb.append(", exprs=").append(TUtil.arrayToString(targets)).append(")"); + sb.append(", exprs=").append(StringUtils.join(targets)).append(")"); } return sb.toString(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java index 7e335b0..ced9a36 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java @@ -43,7 +43,19 @@ public abstract class RelationNode extends LogicalNode { public abstract String getTableName(); + /** + * Return a full qualified table name (i.e., dbname.table_name) + * + * @return A full qualified table name + */ public abstract String getCanonicalName(); + /** + * Return a logical schema, meaning physically stored columns and virtual columns. + * Since partition keys in the column partition are not physically stored to files or tables, + * we call the partition keys virtual columns. + * + * @return A logical schema + */ public abstract Schema getLogicalSchema(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java index 8215f51..7487009 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Column; +import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; import static org.apache.tajo.plan.serder.PlanProto.ShuffleType; @@ -107,7 +108,7 @@ public class ShuffleFileWriteNode extends PersistentStoreNode implements Cloneab } sb.append(", part number=").append(numOutputs); if (shuffleKeys != null) { - sb.append(", keys: ").append(TUtil.arrayToString(shuffleKeys)); + sb.append(", keys: ").append(StringUtils.join(shuffleKeys)); } sb.append(")"); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java index 0166ef8..ef61d51 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java @@ -20,7 +20,7 @@ package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; import org.apache.tajo.plan.PlanString; -import org.apache.tajo.util.TUtil; +import org.apache.tajo.util.StringUtils; import java.util.List; @@ -52,7 +52,7 @@ public class TruncateTableNode extends LogicalNode { @Override public String toString() { - return "TruncateTable (table=" + TUtil.collectionToString(tableNames, ", ") + ")"; + return "TruncateTable (table=" + StringUtils.join(tableNames, ", ") + ")"; } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java index 3f624f6..1bee34c 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java @@ -26,6 +26,7 @@ import org.apache.tajo.plan.PlanString; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.Target; import org.apache.tajo.plan.expr.WindowFunctionEval; +import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; public class WindowAggNode extends UnaryNode implements Projectable, Cloneable { @@ -114,14 +115,14 @@ public class WindowAggNode extends UnaryNode implements Projectable, Cloneable { public String toString() { StringBuilder sb = new StringBuilder("WinAgg ("); if (hasPartitionKeys()) { - sb.append("partition keys=").append(TUtil.arrayToString(partitionKeys)); + sb.append("partition keys=").append(StringUtils.join(partitionKeys)); sb.append(", "); } if (hasAggFunctions()) { - sb.append("funcs=").append(TUtil.arrayToString(windowFuncs)); + sb.append("funcs=").append(StringUtils.join(windowFuncs)); } if (hasSortSpecs()) { - sb.append("sort=").append(TUtil.arrayToString(sortSpecs)); + sb.append("sort=").append(StringUtils.join(sortSpecs)); } sb.append(")"); return sb.toString(); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java index 2c55c81..633b30e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java @@ -18,11 +18,14 @@ package org.apache.tajo.plan.nameresolver; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.tajo.algebra.ColumnReferenceExpr; +import org.apache.tajo.algebra.Relation; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.NestedPathUtil; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.exception.NoSuchColumnException; import org.apache.tajo.plan.algebra.AmbiguousFieldException; @@ -31,14 +34,30 @@ import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.verifier.VerifyException; import org.apache.tajo.plan.logical.RelationNode; import org.apache.tajo.util.Pair; +import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; -import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; /** - * NameResolver utility + * Column name resolution utility. A SQL statement can include many kinds of column names, + * defined in different ways. Some column name indicates just a column in a relation. + * Another column name includes alias table name or alias column name, derived from some expression. + * + * This utility ensures that each column name is derived from valid and accessible column, and + * it also finds the exact data type of the column. + * + * Terminology: + * <ul> + * <li>Qualifier: database name, table name, or both included in a column name</li> + * <li>Simple name: just column name without any qualifier</li> + * <li>Alias name: another name to shortly specify a certain column</li> + * <li>Fully qualified name: a column name with database name and table name</li> + * <li>Canonical name: a fully qualified name, but its simple name is aliased name.</li> + * </ul> */ public abstract class NameResolver { @@ -51,30 +70,44 @@ public abstract class NameResolver { resolverMap.put(NameResolvingMode.LEGACY, new ResolverByLegacy()); } + public static Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr column, + NameResolvingMode mode) throws PlanningException { + if (!resolverMap.containsKey(mode)) { + throw new PlanningException("Unsupported name resolving level: " + mode.name()); + } + return resolverMap.get(mode).resolve(plan, block, column); + } + abstract Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) throws PlanningException; /** - * Try to find the database name + * Guess a relation from a table name regardless of whether the given name is qualified or not. * * @param block the current block - * @param tableName The table name - * @return The found database name + * @param tableName The table name which can be either qualified or not. + * @return A corresponding relation * @throws PlanningException */ - public static String resolveDatabase(LogicalPlan.QueryBlock block, String tableName) throws PlanningException { - List<String> found = new ArrayList<String>(); + public static RelationNode lookupTable(LogicalPlan.QueryBlock block, String tableName) throws PlanningException { + List<RelationNode> found = TUtil.newList(); + for (RelationNode relation : block.getRelations()) { - // check alias name or table name - if (CatalogUtil.extractSimpleName(relation.getCanonicalName()).equals(tableName) || + + // if a table name is qualified + if (relation.getCanonicalName().equals(tableName) || relation.getTableName().equals(tableName)) { + found.add(relation); + + // if a table name is not qualified + } else if (CatalogUtil.extractSimpleName(relation.getCanonicalName()).equals(tableName) || CatalogUtil.extractSimpleName(relation.getTableName()).equals(tableName)) { - // obtain the database name - found.add(CatalogUtil.extractQualifier(relation.getTableName())); + found.add(relation); } } if (found.size() == 0) { return null; + } else if (found.size() > 1) { throw new PlanningException("Ambiguous table name \"" + tableName + "\""); } @@ -82,12 +115,26 @@ public abstract class NameResolver { return found.get(0); } - public static Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr column, - NameResolvingMode mode) throws PlanningException { - if (!resolverMap.containsKey(mode)) { - throw new PlanningException("Unsupported name resolving level: " + mode.name()); + /** + * Find relations such that its schema contains a given column + * + * @param block the current block + * @param columnName The column name to find relation + * @return relations including a given column + * @throws PlanningException + */ + public static Collection<RelationNode> lookupTableByColumns(LogicalPlan.QueryBlock block, String columnName) + throws PlanningException { + + Set<RelationNode> found = TUtil.newHashSet(); + + for (RelationNode rel : block.getRelations()) { + if (rel.getLogicalSchema().contains(columnName)) { + found.add(rel); + } } - return resolverMap.get(mode).resolve(plan, block, column); + + return found; } /** @@ -107,7 +154,7 @@ public abstract class NameResolver { String canonicalName; if (columnRef.hasQualifier()) { - Pair<String, String> normalized = normalizeQualifierAndCanonicalName(block, columnRef); + Pair<String, String> normalized = lookupQualifierAndCanonicalName(block, columnRef); qualifier = normalized.getFirst(); canonicalName = normalized.getSecond(); @@ -121,8 +168,8 @@ public abstract class NameResolver { // Please consider a query case: // select lineitem.l_orderkey from lineitem a order by lineitem.l_orderkey; // - // The relation lineitem is already renamed to "a", but lineitem.l_orderkey still can be used. - // The below code makes it available. Otherwise, it cannot find any match in the relation schema. + // The relation lineitem is already renamed to "a", but lineitem.l_orderkey still should be available. + // The below code makes it possible. Otherwise, it cannot find any match in the relation schema. if (block.isAlreadyRenamedTableName(CatalogUtil.extractQualifier(canonicalName))) { canonicalName = CatalogUtil.buildFQName(relationOp.getCanonicalName(), CatalogUtil.extractSimpleName(canonicalName)); @@ -133,7 +180,7 @@ public abstract class NameResolver { return column; } else { - return resolveFromAllRelsInBlock(block, columnRef); + return lookupColumnFromAllRelsInBlock(block, columnRef.getName()); } } @@ -162,18 +209,22 @@ public abstract class NameResolver { } /** - * It tries to find a full qualified column name from all relations in the current block. + * Lookup a column among all relations in the current block from a column name. + * + * It assumes that <code>columnName</code> is not any qualified name. * * @param block The current query block - * @param columnRef The column reference to be found + * @param columnName The column reference to be found * @return The found column */ - static Column resolveFromAllRelsInBlock(LogicalPlan.QueryBlock block, - ColumnReferenceExpr columnRef) throws VerifyException { + static Column lookupColumnFromAllRelsInBlock(LogicalPlan.QueryBlock block, + String columnName) throws VerifyException { + Preconditions.checkArgument(CatalogUtil.isSimpleIdentifier(columnName)); + List<Column> candidates = TUtil.newList(); for (RelationNode rel : block.getRelations()) { - Column found = rel.getLogicalSchema().getColumn(columnRef.getName()); + Column found = rel.getLogicalSchema().getColumn(columnName); if (found != null) { candidates.add(found); } @@ -240,39 +291,100 @@ public abstract class NameResolver { } /** - * It returns a pair of names, which the first value is ${database}.${table} and the second value - * is a simple column name. + * Lookup a qualifier and a canonical name of column. + * + * It returns a pair of names, which the first value is the qualifier ${database}.${table} and + * the second value is column's simple name. * * @param block The current block * @param columnRef The column name * @return A pair of normalized qualifier and column name * @throws PlanningException */ - static Pair<String, String> normalizeQualifierAndCanonicalName(LogicalPlan.QueryBlock block, - ColumnReferenceExpr columnRef) + static Pair<String, String> lookupQualifierAndCanonicalName(LogicalPlan.QueryBlock block, + ColumnReferenceExpr columnRef) throws PlanningException { - String qualifier; - String canonicalName; + Preconditions.checkArgument(columnRef.hasQualifier(), "ColumnReferenceExpr must be qualified."); + + String [] qualifierParts = columnRef.getQualifier().split("\\."); - if (CatalogUtil.isFQTableName(columnRef.getQualifier())) { - qualifier = columnRef.getQualifier(); - canonicalName = columnRef.getCanonicalName(); + // This method assumes that column name consists of two or more dot chained names. + // In this case, there must be three cases as follows: + // + // - dbname.tbname.column_name.nested_field... + // - tbname.column_name.nested_field... + // - column.nested_fieldX... + + Set<RelationNode> guessedRelations = TUtil.newHashSet(); + + // this position indicates the index of column name in qualifierParts; + // It must be 0 or more because a qualified column is always passed to lookupQualifierAndCanonicalName(). + int columnNamePosition = -1; + + // check for dbname.tbname.column_name.nested_field + if (qualifierParts.length >= 2) { + RelationNode rel = lookupTable(block, CatalogUtil.buildFQName(qualifierParts[0], qualifierParts[1])); + if (rel != null) { + guessedRelations.add(rel); + columnNamePosition = 2; + } + } + + // check for tbname.column_name.nested_field + if (qualifierParts.length >= 1) { + RelationNode rel = lookupTable(block, qualifierParts[0]); + if (rel != null) { + guessedRelations.add(rel); + columnNamePosition = 1; + } + } + + // column.nested_fieldX... + if (guessedRelations.size() == 0 && qualifierParts.length == 1) { + Collection<RelationNode> rels = lookupTableByColumns(block, qualifierParts[0]); + + if (rels.size() > 1) { + throw new AmbiguousFieldException(columnRef.getCanonicalName()); + } + + if (rels.size() == 1) { + guessedRelations.addAll(rels); + columnNamePosition = 0; + } + } + + // throw exception if no column cannot be founded or two or more than columns are founded + if (guessedRelations.size() == 0) { + throw new NoSuchColumnException(columnRef.getQualifier()); + } else if (guessedRelations.size() > 1) { + throw new AmbiguousFieldException(columnRef.getCanonicalName()); + } + + String qualifier = guessedRelations.iterator().next().getCanonicalName(); + String columnName = ""; + + if (columnNamePosition >= qualifierParts.length) { // if there is no column in qualifierParts + columnName = columnRef.getName(); } else { - String resolvedDatabaseName = resolveDatabase(block, columnRef.getQualifier()); - if (resolvedDatabaseName == null) { - throw new NoSuchColumnException(columnRef.getQualifier()); + // join a column name and its nested field names + columnName = qualifierParts[columnNamePosition]; + + // if qualifierParts include nested field names + if (qualifierParts.length > columnNamePosition) { + columnName += StringUtils.join(qualifierParts, "/", columnNamePosition + 1, qualifierParts.length); } - qualifier = CatalogUtil.buildFQName(resolvedDatabaseName, columnRef.getQualifier()); - canonicalName = CatalogUtil.buildFQName(qualifier, columnRef.getName()); + + // columnRef always has a leaf field name. + columnName += "/" + columnRef.getName(); } - return new Pair<String, String>(qualifier, canonicalName); + return new Pair<String, String>(qualifier, columnName); } static Column ensureUniqueColumn(List<Column> candidates) throws VerifyException { if (candidates.size() == 1) { return candidates.get(0); - } else if (candidates.size() > 2) { + } else if (candidates.size() > 1) { StringBuilder sb = new StringBuilder(); boolean first = true; for (Column column : candidates) { http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java index 19f39dd..0a665ab 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java @@ -48,12 +48,10 @@ public class ResolverByLegacy extends NameResolver { private static Column resolveColumnWithQualifier(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) throws PlanningException { final String qualifier; - String canonicalName; final String qualifiedName; - Pair<String, String> normalized = normalizeQualifierAndCanonicalName(block, columnRef); + Pair<String, String> normalized = lookupQualifierAndCanonicalName(block, columnRef); qualifier = normalized.getFirst(); - canonicalName = normalized.getSecond(); qualifiedName = CatalogUtil.buildFQName(qualifier, columnRef.getName()); Column found = resolveFromRelsWithinBlock(plan, block, columnRef); @@ -84,7 +82,7 @@ public class ResolverByLegacy extends NameResolver { && currentNode.getType() != NodeType.TABLE_SUBQUERY) { List<Column> candidates = TUtil.newList(); if (block.getNamedExprsManager().isAliased(qualifiedName)) { - String alias = block.getNamedExprsManager().getAlias(canonicalName); + String alias = block.getNamedExprsManager().getAlias(qualifiedName); found = resolve(plan, block, new ColumnReferenceExpr(alias), NameResolvingMode.LEGACY); if (found != null) { candidates.add(found); @@ -101,7 +99,7 @@ public class ResolverByLegacy extends NameResolver { static Column resolveColumnWithoutQualifier(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef)throws PlanningException { - Column found = resolveFromAllRelsInBlock(block, columnRef); + Column found = lookupColumnFromAllRelsInBlock(block, columnRef.getName()); if (found != null) { return found; } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index dc6b8ef..cc54a22 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@ -726,7 +726,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo // find aggregation column Set<Column> groupingColumns = TUtil.newHashSet(groupByNode.getGroupingColumns()); Set<String> aggrFunctionOutColumns = TUtil.newHashSet(); - for (Column column : groupByNode.getOutSchema().getColumns()) { + for (Column column : groupByNode.getOutSchema().getRootColumns()) { if (!groupingColumns.contains(column)) { aggrFunctionOutColumns.add(column.getQualifiedName()); } @@ -842,7 +842,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo TableDesc table = scanNode.getTableDesc(); boolean hasQualifiedName = false; if (table.hasPartition()) { - for (Column c: table.getPartitionMethod().getExpressionSchema().getColumns()) { + for (Column c: table.getPartitionMethod().getExpressionSchema().getRootColumns()) { partitionColumns.add(c.getQualifiedName()); hasQualifiedName = c.hasQualifier(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java index d8b6380..4b9fd48 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java @@ -224,7 +224,7 @@ public class PartitionedTableRewriter implements LogicalPlanRewriteRule { PartitionMethodDesc partitionDesc = scanNode.getTableDesc().getPartitionMethod(); Schema paritionValuesSchema = new Schema(); - for (Column column : partitionDesc.getExpressionSchema().getColumns()) { + for (Column column : partitionDesc.getExpressionSchema().getRootColumns()) { paritionValuesSchema.addColumn(column); } @@ -237,7 +237,7 @@ public class PartitionedTableRewriter implements LogicalPlanRewriteRule { // add qualifier to schema for qual paritionValuesSchema.setQualifier(scanNode.getCanonicalName()); - for (Column column : paritionValuesSchema.getColumns()) { + for (Column column : paritionValuesSchema.getRootColumns()) { for (EvalNode simpleExpr : conjunctiveForms) { if (checkIfIndexablePredicateOnTargetColumn(simpleExpr, column)) { indexablePredicateSet.add(simpleExpr); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index 763f938..3b83ded 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -146,7 +146,7 @@ public class PlannerUtil { if (EvalTreeUtil.checkIfPartitionSelection(node, partSchema)) { prefixPartitionWhere = true; boolean isPrefix = true; - for (Column c : partSchema.getColumns()) { + for (Column c : partSchema.getRootColumns()) { String value = EvalTreeUtil.getPartitionValue(node, c.getSimpleName()); if (isPrefix && value == null) isPrefix = false; http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java index 22b3351..c5ef8ef 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java @@ -23,6 +23,8 @@ import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.util.PlannerUtil; @@ -55,7 +57,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri /** * It checks if an output schema of a projectable node and target's output data types are equivalent to each other. */ - private static void verifyProjectableOutputSchema(Projectable node) throws PlanningException { + private static void verifyProjectableOutputSchema(Context context, Projectable node) throws PlanningException { Schema outputSchema = node.getOutSchema(); Schema targetSchema = PlannerUtil.targetToSchema(node.getTargets()); @@ -66,9 +68,15 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri } for (int i = 0; i < outputSchema.size(); i++) { - if (!outputSchema.getColumn(i).getDataType().equals(targetSchema.getColumn(i).getDataType())) { + Column outputColumn = outputSchema.getColumn(i); + + if (outputColumn.getDataType().getType() == Type.RECORD) { + context.state.addVerification("Projecting RECORD fields is not supported yet."); + } + + if (!outputColumn.getDataType().equals(targetSchema.getColumn(i).getDataType())) { Column targetColumn = targetSchema.getColumn(i); - Column insertColumn = outputSchema.getColumn(i); + Column insertColumn = outputColumn; throw new PlanningException("ERROR: " + insertColumn.getSimpleName() + " is of type " + insertColumn.getDataType().getType().name() + ", but target column '" + targetColumn.getSimpleName() + "' is of type " + @@ -86,7 +94,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri ExprsVerifier.verify(state.state, node, target.getEvalTree()); } - verifyProjectableOutputSchema(node); + verifyProjectableOutputSchema(state, node); return node; } @@ -108,7 +116,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri GroupbyNode node, Stack<LogicalNode> stack) throws PlanningException { super.visitGroupBy(context, plan, block, node, stack); - verifyProjectableOutputSchema(node); + verifyProjectableOutputSchema(context, node); return node; } @@ -130,7 +138,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri ExprsVerifier.verify(context.state, node, node.getJoinQual()); } - verifyProjectableOutputSchema(node); + verifyProjectableOutputSchema(context, node); return node; } @@ -192,7 +200,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri } } - verifyProjectableOutputSchema(node); + verifyProjectableOutputSchema(context, node); return node; } @@ -209,7 +217,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri ExprsVerifier.verify(context.state, node, node.getQual()); } - verifyProjectableOutputSchema(node); + verifyProjectableOutputSchema(context, node); return node; } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index 275ab3a..8717bda 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -83,7 +83,7 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor<PreLogicalPlanVer super.visitProjection(context, stack, expr); Set<String> names = TUtil.newHashSet(); - + for (NamedExpr namedExpr : expr.getNamedExprs()) { if (namedExpr.hasAlias()) { @@ -302,8 +302,8 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor<PreLogicalPlanVer return null; } if (table.hasPartition()) { - int columnSize = table.getSchema().getColumns().size(); - columnSize += table.getPartitionMethod().getExpressionSchema().getColumns().size(); + int columnSize = table.getSchema().getRootColumns().size(); + columnSize += table.getPartitionMethod().getExpressionSchema().getRootColumns().size(); if (projectColumnNum < columnSize) { context.state.addVerification("INSERT has smaller expressions than target columns"); } else if (projectColumnNum > columnSize) { http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java index 66b3667..5423fd7 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java @@ -84,7 +84,7 @@ public class MergeScanner implements Scanner { tableStats.setNumBytes(numBytes); tableStats.setNumBlocks(fragments.size()); - for(Column eachColumn: schema.getColumns()) { + for(Column eachColumn: schema.getRootColumns()) { ColumnStats columnStats = new ColumnStats(eachColumn); tableStats.addColumnStat(columnStats); } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java index 33db798..9d69423 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java @@ -35,7 +35,7 @@ public class RowStoreUtil { public static int[] getTargetIds(Schema inSchema, Schema outSchema) { int[] targetIds = new int[outSchema.size()]; int i = 0; - for (Column target : outSchema.getColumns()) { + for (Column target : outSchema.getRootColumns()) { targetIds[i] = inSchema.getColumnId(target.getQualifiedName()); i++; } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java index 0356b19..7af8247 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java @@ -61,6 +61,15 @@ public interface Scanner extends SchemaObject, Closeable { /** * It returns if the projection is executed in the underlying scanner layer. * + * If TRUE, the upper layers (i.e., SeqScanExec) assume that next() + * will return a tuple which contains only projected fields. In other words, + * the field number of a retrieved tuple is equivalent tothe number of targets. + * + * If FALSE, the upper layers assume that next() will return a tuple which + * contains projected fields and non-projected fields, padded by NullDatum. + * In other words, the width of tuple is equivalent to the field number + * of the table schema. + * * @return true if this scanner can project the given columns. */ boolean isProjectable(); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java index ce963c8..b8438e9 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java @@ -409,9 +409,7 @@ public abstract class StorageManager { Class<? extends Scanner> scannerClass = getScannerClass(meta.getStoreType()); scanner = newScannerInstance(scannerClass, conf, schema, meta, fragment); - if (scanner.isProjectable()) { - scanner.setTarget(target.toArray()); - } + scanner.setTarget(target.toArray()); return scanner; } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java index 4836352..0e3441b 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java @@ -60,7 +60,6 @@ public abstract class AbstractHBaseAppender implements Appender { protected boolean[] isRowKeyMappings; protected boolean[] isColumnKeys; protected boolean[] isColumnValues; - protected int[] rowKeyFieldIndexes; protected int[] rowkeyColumnIndexes; protected char rowKeyDelimiter; @@ -107,7 +106,6 @@ public abstract class AbstractHBaseAppender implements Appender { isColumnKeys = columnMapping.getIsColumnKeys(); isColumnValues = columnMapping.getIsColumnValues(); rowKeyDelimiter = columnMapping.getRowKeyDelimiter(); - rowKeyFieldIndexes = columnMapping.getRowKeyFieldIndexes(); this.columnNum = schema.size(); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java index c3094fd..e66a707 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java @@ -77,7 +77,7 @@ public class ColumnMapping { String[] columnMappingTokens = columnMapping.split(","); - if (columnMappingTokens.length != schema.getColumns().size()) { + if (columnMappingTokens.length != schema.getRootColumns().size()) { throw new IOException("The number of mapped HBase columns is great than the number of Tajo table columns"); } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java index c05c5bb..53ff9dc 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java @@ -39,10 +39,14 @@ public class HBaseBinarySerializerDeserializer { datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt4(Bytes.toInt(bytes)); break; case INT8: - if (bytes.length == 4) { - datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt8(Bytes.toInt(bytes)); + if (bytes == null) { + datum = NullDatum.get(); } else { - datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt8(Bytes.toLong(bytes)); + if (bytes.length == 4) { + datum = DatumFactory.createInt8(Bytes.toInt(bytes)); + } else { + datum = bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt8(Bytes.toLong(bytes)); + } } break; case FLOAT4: http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java index ab56252..9eb1d86 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java @@ -18,6 +18,7 @@ package org.apache.tajo.storage.hbase; +import com.google.common.base.Preconditions; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -87,7 +88,13 @@ public class HBaseScanner implements Scanner { private char rowKeyDelimiter; public HBaseScanner (Configuration conf, Schema schema, TableMeta meta, Fragment fragment) throws IOException { - this.conf = (TajoConf)conf; + Preconditions.checkNotNull(conf); + Preconditions.checkNotNull(schema); + Preconditions.checkNotNull(meta); + Preconditions.checkNotNull(fragment); + Preconditions.checkArgument(conf instanceof TajoConf); + + this.conf = (TajoConf) conf; this.schema = schema; this.meta = meta; this.fragment = (HBaseFragment)fragment; @@ -102,11 +109,10 @@ public class HBaseScanner implements Scanner { tableStats.setNumBytes(0); tableStats.setNumBlocks(1); } - if (schema != null) { - for(Column eachColumn: schema.getColumns()) { - ColumnStats columnStats = new ColumnStats(eachColumn); - tableStats.addColumnStat(columnStats); - } + + for (Column eachColumn : schema.getRootColumns()) { + ColumnStats columnStats = new ColumnStats(eachColumn); + tableStats.addColumnStat(columnStats); } scanFetchSize = Integer.parseInt( @@ -203,9 +209,9 @@ public class HBaseScanner implements Scanner { } Result result = scanResults[scanResultIndex++]; - Tuple resultTuple = new VTuple(schema.size()); + Tuple resultTuple = new VTuple(targetIndexes.length); for (int i = 0; i < targetIndexes.length; i++) { - resultTuple.put(targetIndexes[i], getDatum(result, targetIndexes[i])); + resultTuple.put(i, getDatum(result, targetIndexes[i])); } numRows++; return resultTuple; http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index a9e5bde..f74245e 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -355,7 +355,7 @@ public class HBaseStorageManager extends StorageManager { Collection<String> columnFamilies = columnMapping.getColumnFamilyNames(); //If 'columns' attribute is empty, Tajo table columns are mapped to all HBase table column. if (columnFamilies.isEmpty()) { - for (Column eachColumn: schema.getColumns()) { + for (Column eachColumn: schema.getRootColumns()) { columnFamilies.add(eachColumn.getSimpleName()); } } @@ -735,7 +735,7 @@ public class HBaseStorageManager extends StorageManager { for (String property : CONNECTION_PROPERTIES) { String thisValue = this.properties.get(property); String thatValue = that.properties.get(property); - //noinspection StringEquality + // noinspection StringEquality if (thisValue == thatValue) { continue; } @@ -1121,7 +1121,7 @@ public class HBaseStorageManager extends StorageManager { try { HTableDescriptor hTableDesc = parseHTableDescriptor(tableMeta, cNode.getTableSchema()); - LOG.info("Delete table cause query failed:" + hTableDesc.getName()); + LOG.info("Delete table cause query failed:" + new String(hTableDesc.getName())); hAdmin.disableTable(hTableDesc.getName()); hAdmin.deleteTable(hTableDesc.getName()); } finally { http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java index c1047d9..5fc96f1 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java @@ -528,7 +528,7 @@ public class CSVFile { @Override public boolean isProjectable() { - return true; + return false; } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java index 038f0f4..0726125 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java @@ -69,7 +69,7 @@ public abstract class FileScanner implements Scanner { } if (schema != null) { - for(Column eachColumn: schema.getColumns()) { + for(Column eachColumn: schema.getRootColumns()) { ColumnStats columnStats = new ColumnStats(eachColumn); tableStats.addColumnStat(columnStats); } http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java index 8d425b4..439bee0 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java @@ -732,7 +732,7 @@ public class FileStorageManager extends StorageManager { //In the case of partitioned table, we should return same partition key data files. int partitionDepth = 0; if (tableDesc.hasPartition()) { - partitionDepth = tableDesc.getPartitionMethod().getExpressionSchema().getColumns().size(); + partitionDepth = tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns().size(); } List<FileStatus> nonZeroLengthFiles = new ArrayList<FileStatus>(); http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java index 51594df..26083a5 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java @@ -80,8 +80,7 @@ public class AvroScanner extends FileScanner { avroSchema = AvroUtil.getAvroSchema(meta, conf); avroFields = avroSchema.getFields(); - DatumReader<GenericRecord> datumReader = - new GenericDatumReader<GenericRecord>(avroSchema); + DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(avroSchema); SeekableInput input = new FsInput(fragment.getPath(), conf); dataFileReader = new DataFileReader<GenericRecord>(input, datumReader); super.init(); @@ -175,13 +174,13 @@ public class AvroScanner extends FileScanner { return null; } - Tuple tuple = new VTuple(schema.size()); + Tuple tuple = new VTuple(projectionMap.length); GenericRecord record = dataFileReader.next(); for (int i = 0; i < projectionMap.length; ++i) { int columnIndex = projectionMap[i]; Object value = record.get(columnIndex); if (value == null) { - tuple.put(columnIndex, NullDatum.get()); + tuple.put(i, NullDatum.get()); continue; } @@ -196,28 +195,28 @@ public class AvroScanner extends FileScanner { TajoDataTypes.Type tajoType = dataType.getType(); switch (avroType) { case NULL: - tuple.put(columnIndex, NullDatum.get()); + tuple.put(i, NullDatum.get()); break; case BOOLEAN: - tuple.put(columnIndex, DatumFactory.createBool((Boolean)value)); + tuple.put(i, DatumFactory.createBool((Boolean)value)); break; case INT: - tuple.put(columnIndex, convertInt(value, tajoType)); + tuple.put(i, convertInt(value, tajoType)); break; case LONG: - tuple.put(columnIndex, DatumFactory.createInt8((Long)value)); + tuple.put(i, DatumFactory.createInt8((Long)value)); break; case FLOAT: - tuple.put(columnIndex, DatumFactory.createFloat4((Float)value)); + tuple.put(i, DatumFactory.createFloat4((Float)value)); break; case DOUBLE: - tuple.put(columnIndex, DatumFactory.createFloat8((Double)value)); + tuple.put(i, DatumFactory.createFloat8((Double)value)); break; case BYTES: - tuple.put(columnIndex, convertBytes(value, tajoType, dataType)); + tuple.put(i, convertBytes(value, tajoType, dataType)); break; case STRING: - tuple.put(columnIndex, convertString(value, tajoType)); + tuple.put(i, convertString(value, tajoType)); break; case RECORD: throw new RuntimeException("Avro RECORD not supported."); @@ -228,7 +227,7 @@ public class AvroScanner extends FileScanner { case UNION: throw new RuntimeException("Avro UNION not supported."); case FIXED: - tuple.put(columnIndex, new BlobDatum(((GenericFixed)value).bytes())); + tuple.put(i, new BlobDatum(((GenericFixed)value).bytes())); break; default: throw new RuntimeException("Unknown type.");
