Repository: tajo Updated Branches: refs/heads/master d7c2c957a -> b37583613
TAJO-884: complex join conditions should be supported in ON clause. Closes #44 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/b3758361 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/b3758361 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/b3758361 Branch: refs/heads/master Commit: b3758361335a4cd26d8b7b9d18879b2b35f612fb Parents: d7c2c95 Author: Hyunsik Choi <[email protected]> Authored: Thu Jun 26 11:41:21 2014 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Thu Jun 26 11:41:21 2014 +0900 ---------------------------------------------------------------------- CHANGES | 3 + .../apache/tajo/engine/eval/EvalTreeUtil.java | 9 +- .../tajo/engine/planner/ExprAnnotator.java | 2 +- .../tajo/engine/planner/ExprNormalizer.java | 44 ++++++-- .../apache/tajo/engine/planner/LogicalPlan.java | 36 ++++--- .../tajo/engine/planner/LogicalPlanner.java | 7 +- .../engine/planner/logical/join/JoinGraph.java | 25 ++++- .../planner/rewrite/ProjectionPushDownRule.java | 108 +++++++++++++++---- .../tajo/engine/query/TestCreateTable.java | 2 +- .../apache/tajo/engine/query/TestJoinQuery.java | 48 +++++++++ .../testLeftOuterJoinWithConstantExpr3.sql | 2 +- .../queries/TestJoinQuery/table1_int4_ddl.sql | 3 + .../queries/TestJoinQuery/table1_int8_ddl.sql | 3 + .../TestJoinQuery/testComplexJoinCondition1.sql | 6 ++ .../TestJoinQuery/testComplexJoinCondition2.sql | 6 ++ .../TestJoinQuery/testComplexJoinCondition3.sql | 6 ++ .../TestJoinQuery/testComplexJoinCondition4.sql | 6 ++ .../testDifferentTypesJoinCondition.sql | 1 + .../testLeftOuterJoinWithConstantExpr3.sql | 2 +- .../testComplexJoinCondition1.result | 27 +++++ .../testComplexJoinCondition2.result | 27 +++++ .../testComplexJoinCondition3.result | 27 +++++ .../testComplexJoinCondition4.result | 29 +++++ .../testDifferentTypesJoinCondition.result | 7 ++ 24 files changed, 379 insertions(+), 57 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 0185044..77186e6 100644 --- a/CHANGES +++ b/CHANGES @@ -74,6 +74,9 @@ Release 0.9.0 - unreleased BUG FIXES + TAJO-884: complex join conditions should be supported in ON clause. + (hyunsik) + TAJO-874: Sometimes InvalidOperationException occurs when aggregates TableStat. (Hyoungjun Kim via hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java index 07f71dc..3921a7d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java @@ -248,14 +248,17 @@ public class EvalTreeUtil { } BinaryEval binaryEval = (BinaryEval) expr; - boolean isBothTermFields = - binaryEval.getLeftExpr().getType() == EvalType.FIELD && - binaryEval.getRightExpr().getType() == EvalType.FIELD; + boolean isBothTermFields = isSingleColumn(binaryEval.getLeftExpr()) && isSingleColumn(binaryEval.getRightExpr()); + return joinComparator && isBothTermFields; } else { return false; } } + + static boolean isSingleColumn(EvalNode evalNode) { + return EvalTreeUtil.findUniqueColumns(evalNode).size() == 1; + } public static class ChangeColumnRefVisitor implements EvalNodeVisitor { private final String findColumn; http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java index 3a53cd2..e143823 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java @@ -152,7 +152,7 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva private static EvalNode convertType(EvalNode evalNode, DataType toType) { // if original and toType is the same, we don't need type conversion. - if (evalNode.getValueType() == toType) { + if (evalNode.getValueType().equals(toType)) { return evalNode; } // the conversion to null is not allowed. http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java index b87665a..75b2b95 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java @@ -20,6 +20,7 @@ package org.apache.tajo.engine.planner; import org.apache.tajo.algebra.*; import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.engine.exception.NoSuchColumnException; import java.util.ArrayList; import java.util.List; @@ -80,15 +81,21 @@ class ExprNormalizer extends SimpleAlgebraVisitor<ExprNormalizer.ExprNormalizedR public static class ExprNormalizedResult { private final LogicalPlan plan; private final LogicalPlan.QueryBlock block; + private final boolean tryBinaryCommonTermsElimination; Expr baseExpr; // outmost expressions, which can includes one or more references of the results of aggregation // function. List<NamedExpr> aggExprs = new ArrayList<NamedExpr>(); // aggregation functions List<NamedExpr> scalarExprs = new ArrayList<NamedExpr>(); // scalar expressions which can be referred - private ExprNormalizedResult(LogicalPlanner.PlanContext context) { + private ExprNormalizedResult(LogicalPlanner.PlanContext context, boolean tryBinaryCommonTermsElimination) { this.plan = context.plan; this.block = context.queryBlock; + this.tryBinaryCommonTermsElimination = tryBinaryCommonTermsElimination; + } + + public boolean isBinaryCommonTermsElimination() { + return tryBinaryCommonTermsElimination; } @Override @@ -98,7 +105,11 @@ class ExprNormalizer extends SimpleAlgebraVisitor<ExprNormalizer.ExprNormalizedR } public ExprNormalizedResult normalize(LogicalPlanner.PlanContext context, Expr expr) throws PlanningException { - ExprNormalizedResult exprNormalizedResult = new ExprNormalizedResult(context); + return normalize(context, expr, false); + } + public ExprNormalizedResult normalize(LogicalPlanner.PlanContext context, Expr expr, boolean subexprElimination) + throws PlanningException { + ExprNormalizedResult exprNormalizedResult = new ExprNormalizedResult(context, subexprElimination); Stack<Expr> stack = new Stack<Expr>(); stack.push(expr); visit(exprNormalizedResult, new Stack<Expr>(), expr); @@ -152,9 +163,27 @@ class ExprNormalizer extends SimpleAlgebraVisitor<ExprNormalizer.ExprNormalizedR return expr; } + private boolean isBinaryCommonTermsElimination(ExprNormalizedResult ctx, Expr expr) { + return ctx.isBinaryCommonTermsElimination() && expr.getType() != OpType.Column + && ctx.block.namedExprsMgr.contains(expr); + } + @Override public Expr visitBinaryOperator(ExprNormalizedResult ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException { - super.visitBinaryOperator(ctx, stack, expr); + stack.push(expr); + + visit(ctx, new Stack<Expr>(), expr.getLeft()); + if (isBinaryCommonTermsElimination(ctx, expr.getLeft())) { + String refName = ctx.block.namedExprsMgr.addExpr(expr.getLeft()); + expr.setLeft(new ColumnReferenceExpr(refName)); + } + + visit(ctx, new Stack<Expr>(), expr.getRight()); + if (isBinaryCommonTermsElimination(ctx, expr.getRight())) { + String refName = ctx.block.namedExprsMgr.addExpr(expr.getRight()); + expr.setRight(new ColumnReferenceExpr(refName)); + } + stack.pop(); //////////////////////// // For Left Term @@ -249,9 +278,12 @@ class ExprNormalizer extends SimpleAlgebraVisitor<ExprNormalizer.ExprNormalizedR throws PlanningException { // if a column reference is not qualified, it finds and sets the qualified column name. if (!(expr.hasQualifier() && CatalogUtil.isFQTableName(expr.getQualifier()))) { - if (!ctx.block.namedExprsMgr.contains(expr.getCanonicalName())) { - String normalized = ctx.plan.getNormalizedColumnName(ctx.block, expr); - expr.setName(normalized); + if (!ctx.block.namedExprsMgr.contains(expr.getCanonicalName()) && expr.getType() == OpType.Column) { + try { + String normalized = ctx.plan.getNormalizedColumnName(ctx.block, expr); + expr.setName(normalized); + } catch (NoSuchColumnException nsc) { + } } } return expr; http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java index 0508bac..92df760 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java @@ -26,6 +26,7 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.engine.eval.EvalNode; +import org.apache.tajo.engine.exception.AmbiguousFieldException; import org.apache.tajo.engine.exception.NoSuchColumnException; import org.apache.tajo.engine.exception.VerifyException; import org.apache.tajo.engine.planner.graph.DirectedGraphCursor; @@ -384,6 +385,21 @@ public class LogicalPlan { private Column resolveColumnWithoutQualifier(QueryBlock block, ColumnReferenceExpr columnRef)throws PlanningException { + + List<Column> candidates = TUtil.newList(); + + // It tries to find a full qualified column name from all relations in the current block. + for (RelationNode rel : block.getRelations()) { + Column found = rel.getTableSchema().getColumn(columnRef.getName()); + if (found != null) { + candidates.add(found); + } + } + + if (!candidates.isEmpty()) { + return ensureUniqueColumn(candidates); + } + // Trying to find the column within the current block if (block.currentNode != null && block.currentNode.getInSchema() != null) { Column found = block.currentNode.getInSchema().getColumn(columnRef.getCanonicalName()); @@ -399,7 +415,7 @@ public class LogicalPlan { } } - List<Column> candidates = TUtil.newList(); + // Trying to find columns from aliased references. if (block.namedExprsMgr.isAliased(columnRef.getCanonicalName())) { String originalName = block.namedExprsMgr.getAlias(columnRef.getCanonicalName()); @@ -412,22 +428,10 @@ public class LogicalPlan { return ensureUniqueColumn(candidates); } - // Trying to find columns from other relations in the current block - for (RelationNode rel : block.getRelations()) { - Column found = rel.getTableSchema().getColumn(columnRef.getName()); - if (found != null) { - candidates.add(found); - } - } - - if (!candidates.isEmpty()) { - return ensureUniqueColumn(candidates); - } - // This is an exception case. It means that there are some bugs in other parts. LogicalNode blockRootNode = block.getRoot(); if (blockRootNode != null && blockRootNode.getOutSchema().getColumn(columnRef.getCanonicalName()) != null) { - throw new VerifyException("ERROR: no such a column name "+ columnRef.getCanonicalName()); + throw new NoSuchColumnException("ERROR: no such a column name "+ columnRef.getCanonicalName()); } // Trying to find columns from other relations in other blocks @@ -456,7 +460,7 @@ public class LogicalPlan { return ensureUniqueColumn(candidates); } - throw new VerifyException("ERROR: no such a column name "+ columnRef.getCanonicalName()); + throw new NoSuchColumnException("ERROR: no such a column name "+ columnRef.getCanonicalName()); } private static Column ensureUniqueColumn(List<Column> candidates) @@ -474,7 +478,7 @@ public class LogicalPlan { } sb.append(column); } - throw new VerifyException("Ambiguous Column Name: " + sb.toString()); + throw new AmbiguousFieldException("Ambiguous Column Name: " + sb.toString()); } else { return null; } http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index c9948d8..be7bce6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -767,7 +767,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex QueryBlock block = context.queryBlock; if (join.hasQual()) { - ExprNormalizedResult normalizedResult = normalizer.normalize(context, join.getQual()); + ExprNormalizedResult normalizedResult = normalizer.normalize(context, join.getQual(), true); block.namedExprsMgr.addExpr(normalizedResult.baseExpr); if (normalizedResult.aggExprs.size() > 0 || normalizedResult.scalarExprs.size() > 0) { throw new VerifyException("Filter condition cannot include aggregation function"); @@ -1207,13 +1207,10 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex // See PreLogicalPlanVerifier.visitInsert. // It guarantees that the equivalence between the numbers of target and projected columns. - ScanNode scanNode = context.plan.createNode(ScanNode.class); - scanNode.init(desc); - context.queryBlock.addRelation(scanNode); String [] targets = expr.getTargetColumns(); Schema targetColumns = new Schema(); for (int i = 0; i < targets.length; i++) { - Column targetColumn = context.plan.resolveColumn(context.queryBlock, new ColumnReferenceExpr(targets[i])); + Column targetColumn = desc.getLogicalSchema().getColumn(targets[i]); targetColumns.addColumn(targetColumn); } insertNode.setTargetSchema(targetColumns); http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java index 6e321f4..6390a77 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java @@ -32,6 +32,7 @@ import org.apache.tajo.engine.planner.PlannerUtil; import org.apache.tajo.engine.planner.PlanningException; import org.apache.tajo.engine.planner.graph.SimpleUndirectedGraph; import org.apache.tajo.engine.planner.logical.JoinNode; +import org.apache.tajo.engine.planner.logical.RelationNode; import org.apache.tajo.util.TUtil; import java.util.*; @@ -59,7 +60,17 @@ public class JoinGraph extends SimpleUndirectedGraph<String, JoinEdge> { String qualifier = CatalogUtil.extractQualifier(columnName); relationNames[0] = qualifier; } else { - throw new PlanningException("Cannot expect a referenced relation: " + leftExpr); + // search for a relation which evaluates a right term included in a join condition + for (RelationNode rel : block.getRelations()) { + if (rel.getOutSchema().contains(leftExpr)) { + String qualifier = rel.getCanonicalName(); + relationNames[0] = qualifier; + } + } + + if (relationNames[0] == null) { // if not found + throw new PlanningException("Cannot expect a referenced relation: " + leftExpr); + } } } @@ -71,7 +82,17 @@ public class JoinGraph extends SimpleUndirectedGraph<String, JoinEdge> { String qualifier = CatalogUtil.extractQualifier(columnName); relationNames[1] = qualifier; } else { - throw new PlanningException("Cannot expect a referenced relation: " + rightExpr); + // search for a relation which evaluates a right term included in a join condition + for (RelationNode rel : block.getRelations()) { + if (rel.getOutSchema().contains(rightExpr)) { + String qualifier = rel.getCanonicalName(); + relationNames[1] = qualifier; + } + } + + if (relationNames[1] == null) { // if not found + throw new PlanningException("Cannot expect a referenced relation: " + rightExpr); + } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java index 8e91dca..827daee 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java @@ -135,8 +135,10 @@ public class ProjectionPushDownRule extends private BiMap<Integer, EvalNode> idToEvalBiMap; /** Map: Id -> Names */ private LinkedHashMap<Integer, List<String>> idToNamesMap; - /** Map: Name -> Boolean */ - private LinkedHashMap<String, Boolean> evaluationStateMap; + /** Map: Id -> Boolean */ + private LinkedHashMap<Integer, Boolean> evaluationStateMap; + /** Map: alias name -> Id */ + private LinkedHashMap<String, Integer> aliasMap; private LogicalPlan plan; @@ -146,6 +148,7 @@ public class ProjectionPushDownRule extends idToEvalBiMap = HashBiMap.create(); idToNamesMap = Maps.newLinkedHashMap(); evaluationStateMap = Maps.newLinkedHashMap(); + aliasMap = Maps.newLinkedHashMap(); } private int getNextSeqId() { @@ -153,6 +156,28 @@ public class ProjectionPushDownRule extends } /** + * If some expression is duplicated, we call an alias indicating the duplicated expression 'native alias'. + * This method checks whether a reference is native alias or not. + * + * @param name The reference name + * @return True if the reference is native alias. Otherwise, it will return False. + */ + public boolean isNativeAlias(String name) { + return aliasMap.containsKey(name); + } + + /** + * This method retrieves the name indicating actual expression that an given alias indicate. + * + * @param name an alias name + * @return Real reference name + */ + public String getRealReferenceName(String name) { + int refId = aliasMap.get(name); + return getPrimaryName(refId); + } + + /** * Add an expression with a specified name, which is usually an alias. * Later, you can refer this expression by the specified name. */ @@ -163,12 +188,19 @@ public class ProjectionPushDownRule extends if (nameToIdBiMap.containsKey(specifiedName)) { int refId = nameToIdBiMap.get(specifiedName); EvalNode found = idToEvalBiMap.get(refId); - if (found != null && !evalNode.equals(found)) { - if (found.getType() != EvalType.FIELD && evalNode.getType() != EvalType.FIELD) { - throw new PlanningException("Duplicate alias: " + evalNode); - } - if (found.getType() == EvalType.FIELD) { - idToEvalBiMap.forcePut(refId, evalNode); + if (found != null) { + if (evalNode.equals(found)) { // if input expression already exists + return specifiedName; + } else { + // The case where if existing reference name and a given reference name are the same to each other and + // existing EvalNode and a given EvalNode is the different + if (found.getType() != EvalType.FIELD && evalNode.getType() != EvalType.FIELD) { + throw new PlanningException("Duplicate alias: " + evalNode); + } + + if (found.getType() == EvalType.FIELD) { + idToEvalBiMap.forcePut(refId, evalNode); + } } } } @@ -176,18 +208,19 @@ public class ProjectionPushDownRule extends int refId; if (idToEvalBiMap.inverse().containsKey(evalNode)) { refId = idToEvalBiMap.inverse().get(evalNode); + aliasMap.put(specifiedName, refId); + } else { refId = getNextSeqId(); idToEvalBiMap.put(refId, evalNode); + TUtil.putToNestedList(idToNamesMap, refId, specifiedName); + for (Column column : EvalTreeUtil.findUniqueColumns(evalNode)) { + add(new FieldEval(column)); + } + evaluationStateMap.put(refId, false); } nameToIdBiMap.put(specifiedName, refId); - TUtil.putToNestedList(idToNamesMap, refId, specifiedName); - evaluationStateMap.put(specifiedName, false); - - for (Column column : EvalTreeUtil.findUniqueColumns(evalNode)) { - add(new FieldEval(column)); - } return specifiedName; } @@ -287,7 +320,8 @@ public class ProjectionPushDownRule extends if (!nameToIdBiMap.containsKey(name)) { throw new RuntimeException("No Such target name: " + name); } - return evaluationStateMap.get(name); + int refId = nameToIdBiMap.get(name); + return evaluationStateMap.get(refId); } public void markAsEvaluated(Target target) { @@ -296,7 +330,7 @@ public class ProjectionPushDownRule extends if (!idToNamesMap.containsKey(refId)) { throw new RuntimeException("No such eval: " + evalNode); } - evaluationStateMap.put(target.getCanonicalName(), true); + evaluationStateMap.put(refId, true); } public Iterator<Target> getFilteredTargets(Set<String> required) { @@ -305,6 +339,7 @@ public class ProjectionPushDownRule extends class FilteredTargetIterator implements Iterator<Target> { List<Target> filtered = TUtil.newList(); + Iterator<Target> iterator; public FilteredTargetIterator(Set<String> required) { for (String name : nameToIdBiMap.keySet()) { @@ -312,16 +347,17 @@ public class ProjectionPushDownRule extends filtered.add(getTarget(name)); } } + iterator = filtered.iterator(); } @Override public boolean hasNext() { - return false; + return iterator.hasNext(); } @Override public Target next() { - return null; + return iterator.next(); } @Override @@ -412,8 +448,15 @@ public class ProjectionPushDownRule extends for (String referenceName : referenceNames) { Target target = context.targetListMgr.getTarget(referenceName); - if (context.targetListMgr.isEvaluated(referenceName)) { - finalTargets.add(new Target(new FieldEval(target.getNamedColumn()))); + if (target.getEvalTree().getType() == EvalType.CONST) { + finalTargets.add(target); + } else if (context.targetListMgr.isEvaluated(referenceName)) { + if (context.targetListMgr.isNativeAlias(referenceName)) { + String realRefName = context.targetListMgr.getRealReferenceName(referenceName); + finalTargets.add(new Target(new FieldEval(realRefName, target.getDataType()), referenceName)); + } else { + finalTargets.add(new Target(new FieldEval(target.getNamedColumn()))); + } } else if (LogicalPlanner.checkIfBeEvaluatedAtThis(target.getEvalTree(), node)) { finalTargets.add(target); context.targetListMgr.markAsEvaluated(target); @@ -687,12 +730,35 @@ public class ProjectionPushDownRule extends return node; } + private static void pushDownIfComplexTermInJoinCondition(Context ctx, EvalNode cnf, EvalNode term) + throws PlanningException { + + // If one of both terms in a binary operator is a complex expression, the binary operator will require + // multiple phases. In this case, join cannot evaluate a binary operator. + // So, we should prevent dividing the binary operator into more subexpressions. + if (term.getType() != EvalType.FIELD && !(term instanceof BinaryEval)) { + String refName = ctx.addExpr(term); + EvalTreeUtil.replace(cnf, term, new FieldEval(refName, term.getValueType())); + } + } + public LogicalNode visitJoin(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode node, Stack<LogicalNode> stack) throws PlanningException { Context newContext = new Context(context); String joinQualReference = null; if (node.hasJoinQual()) { + for (EvalNode eachQual : AlgebraicUtil.toConjunctiveNormalFormArray(node.getJoinQual())) { + if (eachQual instanceof BinaryEval) { + BinaryEval binaryQual = (BinaryEval) eachQual; + + for (int i = 0; i < 2; i++) { + EvalNode term = binaryQual.getExpr(i); + pushDownIfComplexTermInJoinCondition(newContext, eachQual, term); + } + } + } + joinQualReference = newContext.addExpr(node.getJoinQual()); newContext.addNecessaryReferences(node.getJoinQual()); } @@ -874,7 +940,7 @@ public class ProjectionPushDownRule extends newContext.addExpr(target); } - for (Iterator<Target> it = getFilteredTarget(targets, context.requiredSet); it.hasNext();) { + for (Iterator<Target> it = context.targetListMgr.getFilteredTargets(newContext.requiredSet); it.hasNext();) { Target target = it.next(); if (LogicalPlanner.checkIfBeEvaluatedAtRelation(block, target.getEvalTree(), node)) { http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java index adc44e3..3d90a79 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java @@ -499,7 +499,7 @@ public class TestCreateTable extends QueryTestCaseBase { // Table created using CTAS executeString("CREATE TABLE table3 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int);").close(); - executeString("CREATE TABLE table4 AS SELECT c1*c1, c2, c2,c3 from table3;").close(); + executeString("CREATE TABLE table4 AS SELECT c1*c1, c2, c2 as c2_a,c3 from table3;").close(); executeString("CREATE TABLE table2 LIKE table4"); testMsg = "testCreateTableLike1: Table using CTAS test failed"; assertTrue(testMsg, isClonedTable("table4","table2")); http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java index 100b9ff..08d4503 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java @@ -856,4 +856,52 @@ public class TestJoinQuery extends QueryTestCaseBase { executeString("DROP TABLE table14 PURGE;"); executeString("DROP TABLE table15 PURGE;"); } + + @Test + public void testDifferentTypesJoinCondition() throws Exception { + // select * from table20 t3 join table21 t4 on t3.id = t4.id; + executeDDL("table1_int8_ddl.sql", "table1", "table20"); + executeDDL("table1_int4_ddl.sql", "table1", "table21"); + try { + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } finally { + executeString("DROP TABLE table20"); + executeString("DROP TABLE table21"); + } + } + + @Test + public void testComplexJoinCondition1() throws Exception { + // select n1.n_nationkey, n1.n_name, n2.n_name from nation n1 join nation n2 on n1.n_name = upper(n2.n_name); + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public void testComplexJoinCondition2() throws Exception { + // select n1.n_nationkey, n1.n_name, upper(n2.n_name) name from nation n1 join nation n2 + // on n1.n_name = upper(n2.n_name); + + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public void testComplexJoinCondition3() throws Exception { + // select n1.n_nationkey, n1.n_name, n2.n_name from nation n1 join nation n2 on lower(n1.n_name) = lower(n2.n_name); + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public void testComplexJoinCondition4() throws Exception { + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr3.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr3.sql b/tajo-core/src/test/resources/queries/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr3.sql index 90be13b..f79b18b 100644 --- a/tajo-core/src/test/resources/queries/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr3.sql +++ b/tajo-core/src/test/resources/queries/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr3.sql @@ -14,4 +14,4 @@ left outer join ( b on a.c_custkey = b.c_custkey order by - c_custkey; \ No newline at end of file + a.c_custkey; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int4_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int4_ddl.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int4_ddl.sql new file mode 100644 index 0000000..0d35cee --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int4_ddl.sql @@ -0,0 +1,3 @@ +create external table ${0} (id int, name text, score float, type text) using csv +with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path}; + http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int8_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int8_ddl.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int8_ddl.sql new file mode 100644 index 0000000..3a7a44a --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int8_ddl.sql @@ -0,0 +1,3 @@ +create external table ${0} (id bigint, name text, score float, type text) using csv +with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path}; + http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition1.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition1.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition1.sql new file mode 100644 index 0000000..b61ad38 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition1.sql @@ -0,0 +1,6 @@ +select + n1.n_nationkey, + n1.n_name, + n2.n_name +from nation n1 join nation n2 on n1.n_name = upper(n2.n_name) +order by n1.n_nationkey; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition2.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition2.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition2.sql new file mode 100644 index 0000000..33effbb --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition2.sql @@ -0,0 +1,6 @@ +select + n1.n_nationkey, + n1.n_name, + upper(n2.n_name) as name +from nation n1 join nation n2 on n1.n_name = upper(n2.n_name) +order by n1.n_nationkey; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition3.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition3.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition3.sql new file mode 100644 index 0000000..5674269 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition3.sql @@ -0,0 +1,6 @@ +select + n1.n_nationkey, + n1.n_name, + n2.n_name +from nation n1 join nation n2 on lower(n1.n_name) = lower(n2.n_name) +order by n1.n_nationkey; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition4.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition4.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition4.sql new file mode 100644 index 0000000..45d8adf --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition4.sql @@ -0,0 +1,6 @@ +select + n1.n_nationkey, + substr(n1.n_name, 1, 4) name1, + substr(n2.n_name, 1, 4) name2 +from nation n1 join nation n2 on substr(n1.n_name, 1, 4) = substr(n2.n_name, 1, 4) +order by n1.n_nationkey; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinQuery/testDifferentTypesJoinCondition.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testDifferentTypesJoinCondition.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testDifferentTypesJoinCondition.sql new file mode 100644 index 0000000..6bd0a4c --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testDifferentTypesJoinCondition.sql @@ -0,0 +1 @@ +select * from table20 t3 join table21 t4 on t3.id = t4.id; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithConstantExpr3.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithConstantExpr3.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithConstantExpr3.sql index 90be13b..f79b18b 100644 --- a/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithConstantExpr3.sql +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithConstantExpr3.sql @@ -14,4 +14,4 @@ left outer join ( b on a.c_custkey = b.c_custkey order by - c_custkey; \ No newline at end of file + a.c_custkey; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition1.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition1.result b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition1.result new file mode 100644 index 0000000..e0691a7 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition1.result @@ -0,0 +1,27 @@ +n_nationkey,n_name,n_name +------------------------------- +0,ALGERIA,ALGERIA +1,ARGENTINA,ARGENTINA +2,BRAZIL,BRAZIL +3,CANADA,CANADA +4,EGYPT,EGYPT +5,ETHIOPIA,ETHIOPIA +6,FRANCE,FRANCE +7,GERMANY,GERMANY +8,INDIA,INDIA +9,INDONESIA,INDONESIA +10,IRAN,IRAN +11,IRAQ,IRAQ +12,JAPAN,JAPAN +13,JORDAN,JORDAN +14,KENYA,KENYA +15,MOROCCO,MOROCCO +16,MOZAMBIQUE,MOZAMBIQUE +17,PERU,PERU +18,CHINA,CHINA +19,ROMANIA,ROMANIA +20,SAUDI ARABIA,SAUDI ARABIA +21,VIETNAM,VIETNAM +22,RUSSIA,RUSSIA +23,UNITED KINGDOM,UNITED KINGDOM +24,UNITED STATES,UNITED STATES \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition2.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition2.result b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition2.result new file mode 100644 index 0000000..63289e1 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition2.result @@ -0,0 +1,27 @@ +n_nationkey,n_name,name +------------------------------- +0,ALGERIA,ALGERIA +1,ARGENTINA,ARGENTINA +2,BRAZIL,BRAZIL +3,CANADA,CANADA +4,EGYPT,EGYPT +5,ETHIOPIA,ETHIOPIA +6,FRANCE,FRANCE +7,GERMANY,GERMANY +8,INDIA,INDIA +9,INDONESIA,INDONESIA +10,IRAN,IRAN +11,IRAQ,IRAQ +12,JAPAN,JAPAN +13,JORDAN,JORDAN +14,KENYA,KENYA +15,MOROCCO,MOROCCO +16,MOZAMBIQUE,MOZAMBIQUE +17,PERU,PERU +18,CHINA,CHINA +19,ROMANIA,ROMANIA +20,SAUDI ARABIA,SAUDI ARABIA +21,VIETNAM,VIETNAM +22,RUSSIA,RUSSIA +23,UNITED KINGDOM,UNITED KINGDOM +24,UNITED STATES,UNITED STATES \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition3.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition3.result b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition3.result new file mode 100644 index 0000000..e0691a7 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition3.result @@ -0,0 +1,27 @@ +n_nationkey,n_name,n_name +------------------------------- +0,ALGERIA,ALGERIA +1,ARGENTINA,ARGENTINA +2,BRAZIL,BRAZIL +3,CANADA,CANADA +4,EGYPT,EGYPT +5,ETHIOPIA,ETHIOPIA +6,FRANCE,FRANCE +7,GERMANY,GERMANY +8,INDIA,INDIA +9,INDONESIA,INDONESIA +10,IRAN,IRAN +11,IRAQ,IRAQ +12,JAPAN,JAPAN +13,JORDAN,JORDAN +14,KENYA,KENYA +15,MOROCCO,MOROCCO +16,MOZAMBIQUE,MOZAMBIQUE +17,PERU,PERU +18,CHINA,CHINA +19,ROMANIA,ROMANIA +20,SAUDI ARABIA,SAUDI ARABIA +21,VIETNAM,VIETNAM +22,RUSSIA,RUSSIA +23,UNITED KINGDOM,UNITED KINGDOM +24,UNITED STATES,UNITED STATES \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition4.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition4.result b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition4.result new file mode 100644 index 0000000..325375d --- /dev/null +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition4.result @@ -0,0 +1,29 @@ +n_nationkey,name1,name2 +------------------------------- +0,ALGE,ALGE +1,ARGE,ARGE +2,BRAZ,BRAZ +3,CANA,CANA +4,EGYP,EGYP +5,ETHI,ETHI +6,FRAN,FRAN +7,GERM,GERM +8,INDI,INDI +9,INDO,INDO +10,IRAN,IRAN +11,IRAQ,IRAQ +12,JAPA,JAPA +13,JORD,JORD +14,KENY,KENY +15,MORO,MORO +16,MOZA,MOZA +17,PERU,PERU +18,CHIN,CHIN +19,ROMA,ROMA +20,SAUD,SAUD +21,VIET,VIET +22,RUSS,RUSS +23,UNIT,UNIT +23,UNIT,UNIT +24,UNIT,UNIT +24,UNIT,UNIT \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/results/TestJoinQuery/testDifferentTypesJoinCondition.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testDifferentTypesJoinCondition.result b/tajo-core/src/test/resources/results/TestJoinQuery/testDifferentTypesJoinCondition.result new file mode 100644 index 0000000..d5b7510 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testDifferentTypesJoinCondition.result @@ -0,0 +1,7 @@ +id,name,score,type,id,name,score,type +------------------------------- +1,ooo,1.1,a,1,ooo,1.1,a +2,ppp,2.3,b,2,ppp,2.3,b +3,qqq,3.4,c,3,qqq,3.4,c +4,rrr,4.5,d,4,rrr,4.5,d +5,xxx,5.6,e,5,xxx,5.6,e \ No newline at end of file
