Repository: tajo Updated Branches: refs/heads/branch-0.11.1 94d02005d -> 04d6d8e80
TAJO-2013: FilterPushDownRule fails due to the unsupported index. Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/04d6d8e8 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/04d6d8e8 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/04d6d8e8 Branch: refs/heads/branch-0.11.1 Commit: 04d6d8e8057d462223ab162c9c50b73506f40dae Parents: 94d0200 Author: Jinho Kim <[email protected]> Authored: Thu Dec 10 14:39:26 2015 +0900 Committer: Jinho Kim <[email protected]> Committed: Thu Dec 10 14:39:26 2015 +0900 ---------------------------------------------------------------------- CHANGES | 2 + .../plan/rewrite/rules/FilterPushDownRule.java | 95 ++++++++++---------- 2 files changed, 52 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/04d6d8e8/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index f7aba71..d55e4b8 100644 --- a/CHANGES +++ b/CHANGES @@ -24,6 +24,8 @@ Release 0.11.1 - unreleased BUG FIXES + TAJO-2013: FilterPushDownRule fails due to the unsupported index. (jinho) + TAJO-2014: TestRpcClientManager fails occasionally. (jinho) TAJO-2000: BSTIndex can cause OOM. (jinho) http://git-wip-us.apache.org/repos/asf/tajo/blob/04d6d8e8/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index e44bba5..1ac29ef 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.catalog.*; import org.apache.tajo.datum.Datum; @@ -57,7 +58,8 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo private CatalogService catalog; static class FilterPushDownContext { - Set<EvalNode> pushingDownFilters = TUtil.newHashSet(); + Set<EvalNode> pushingDownFilters = new HashSet<>(); + LogicalPlanRewriteRuleContext rewriteRuleContext; public void clear() { pushingDownFilters.clear(); @@ -111,6 +113,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo . It not, create new HavingNode and set parent's child. */ FilterPushDownContext context = new FilterPushDownContext(); + context.rewriteRuleContext = rewriteRuleContext; LogicalPlan plan = rewriteRuleContext.getPlan(); catalog = rewriteRuleContext.getCatalog(); for (LogicalPlan.QueryBlock block : plan.getQueryBlocks()) { @@ -129,7 +132,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo @Override public LogicalNode visitFilter(FilterPushDownContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, SelectionNode selNode, Stack<LogicalNode> stack) throws TajoException { - context.pushingDownFilters.addAll(TUtil.newHashSet(AlgebraicUtil.toConjunctiveNormalFormArray(selNode.getQual()))); + context.pushingDownFilters.addAll(new HashSet<>(Arrays.asList(AlgebraicUtil.toConjunctiveNormalFormArray(selNode.getQual())))); stack.push(selNode); visit(context, plan, block, selNode.getChild(), stack); @@ -147,7 +150,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo } else { // if there remain search conditions // check if it can be evaluated here - Set<EvalNode> matched = TUtil.newHashSet(); + Set<EvalNode> matched = new HashSet<>(); for (EvalNode eachEval : context.pushingDownFilters) { if (LogicalPlanner.checkIfBeEvaluatedAtThis(eachEval, selNode)) { matched.add(eachEval); @@ -168,9 +171,9 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo @Override public LogicalNode visitJoin(FilterPushDownContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode joinNode, Stack<LogicalNode> stack) throws TajoException { - Set<EvalNode> onPredicates = TUtil.newHashSet(); + Set<EvalNode> onPredicates = new HashSet<>(); if (joinNode.hasJoinQual()) { - onPredicates.addAll(TUtil.newHashSet(AlgebraicUtil.toConjunctiveNormalFormArray(joinNode.getJoinQual()))); + onPredicates.addAll(new HashSet<>(Arrays.asList(AlgebraicUtil.toConjunctiveNormalFormArray(joinNode.getJoinQual())))); } // clear join qual joinNode.clearJoinQual(); @@ -293,7 +296,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo final Set<EvalNode> onPredicates, final Set<EvalNode> wherePredicates) throws TajoException { - Set<EvalNode> nonPushableQuals = TUtil.newHashSet(); + Set<EvalNode> nonPushableQuals = new HashSet<>(); // TODO: non-equi theta join quals must not be pushed until TAJO-742 is resolved. nonPushableQuals.addAll(extractNonEquiThetaJoinQuals(wherePredicates, block, joinNode)); nonPushableQuals.addAll(extractNonEquiThetaJoinQuals(onPredicates, block, joinNode)); @@ -340,8 +343,8 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo final Set<EvalNode> onPredicates, final Set<EvalNode> wherePredicates, final JoinNode joinNode) throws TajoException { - Set<String> nullSupplyingTableNameSet = TUtil.newHashSet(); - Set<String> preservedTableNameSet = TUtil.newHashSet(); + Set<String> nullSupplyingTableNameSet = new HashSet<>(); + Set<String> preservedTableNameSet = new HashSet<>(); String leftRelation = PlannerUtil.getTopRelationInLineage(plan, joinNode.getLeftChild()); String rightRelation = PlannerUtil.getTopRelationInLineage(plan, joinNode.getRightChild()); @@ -359,7 +362,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo nullSupplyingTableNameSet.add(rightRelation); } - Set<EvalNode> nonPushableQuals = TUtil.newHashSet(); + Set<EvalNode> nonPushableQuals = new HashSet<>(); for (EvalNode eachQual : onPredicates) { for (String relName : preservedTableNameSet) { if (isEvalNeedRelation(eachQual, relName)) { @@ -425,7 +428,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo LogicalPlan.QueryBlock block, LogicalNode node, LogicalNode childNode) throws TajoException { // transformed -> pushingDownFilters - Map<EvalNode, EvalNode> transformedMap = TUtil.newHashMap(); + Map<EvalNode, EvalNode> transformedMap = new HashMap<>(); if (originEvals.isEmpty()) { return transformedMap; @@ -483,7 +486,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo } // node in column -> child out column - Map<String, String> columnMap = TUtil.newHashMap(); + Map<String, String> columnMap = new HashMap<>(); for (int i = 0; i < node.getInSchema().size(); i++) { String inColumnName = node.getInSchema().getColumn(i).getQualifiedName(); @@ -643,7 +646,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo } private Collection<EvalNode> reverseTransform(BiMap<EvalNode, EvalNode> map, Set<EvalNode> remainFilters) { - Set<EvalNode> reversed = TUtil.newHashSet(); + Set<EvalNode> reversed = new HashSet<>(); for (EvalNode evalNode : remainFilters) { reversed.add(map.get(evalNode)); } @@ -655,7 +658,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo LogicalNode childNode, List<EvalNode> notMatched, Set<String> partitionColumns, int columnOffset) throws TajoException { // canonical name -> target - Map<String, Target> nodeTargetMap = TUtil.newHashMap(); + Map<String, Target> nodeTargetMap = new HashMap<>(); for (Target target : node.getTargets()) { nodeTargetMap.put(target.getCanonicalName(), target); } @@ -780,8 +783,8 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo HavingNode havingNode, GroupbyNode groupByNode) throws TajoException { // find aggregation column - Set<Column> groupingColumns = TUtil.newHashSet(groupByNode.getGroupingColumns()); - Set<String> aggrFunctionOutColumns = TUtil.newHashSet(); + Set<Column> groupingColumns = new HashSet<>(Arrays.asList(groupByNode.getGroupingColumns())); + Set<String> aggrFunctionOutColumns = new HashSet<>(); for (Column column : groupByNode.getOutSchema().getRootColumns()) { if (!groupingColumns.contains(column)) { aggrFunctionOutColumns.add(column.getQualifiedName()); @@ -896,7 +899,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo List<EvalNode> matched = TUtil.newList(); // find partition column and check matching - Set<String> partitionColumns = TUtil.newHashSet(); + Set<String> partitionColumns = new HashSet<>(); TableDesc table = scanNode.getTableDesc(); boolean hasQualifiedName = false; if (table.hasPartition()) { @@ -905,7 +908,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo hasQualifiedName = c.hasQualifier(); } } - Set<EvalNode> partitionEvals = TUtil.newHashSet(); + Set<EvalNode> partitionEvals = new HashSet<>(); for (EvalNode eval : context.pushingDownFilters) { if (table.hasPartition()) { Set<Column> columns = EvalTreeUtil.findUniqueColumns(eval); @@ -966,35 +969,37 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo scanNode.setQual(qual); // Index path can be identified only after filters are pushed into each scan. - String databaseName, tableName; - databaseName = CatalogUtil.extractQualifier(table.getName()); - tableName = CatalogUtil.extractSimpleName(table.getName()); - Set<Predicate> predicates = TUtil.newHashSet(); - for (EvalNode eval : PlannerUtil.getAllEqualEvals(qual)) { - BinaryEval binaryEval = (BinaryEval) eval; - // TODO: consider more complex predicates - if (binaryEval.getLeftExpr().getType() == EvalType.FIELD && - binaryEval.getRightExpr().getType() == EvalType.CONST) { - predicates.add(new Predicate(binaryEval.getType(), - ((FieldEval) binaryEval.getLeftExpr()).getColumnRef(), - ((ConstEval)binaryEval.getRightExpr()).getValue())); - } else if (binaryEval.getLeftExpr().getType() == EvalType.CONST && - binaryEval.getRightExpr().getType() == EvalType.FIELD) { - predicates.add(new Predicate(binaryEval.getType(), - ((FieldEval) binaryEval.getRightExpr()).getColumnRef(), - ((ConstEval)binaryEval.getLeftExpr()).getValue())); + if(context.rewriteRuleContext.getQueryContext().getBool(SessionVars.INDEX_ENABLED)) { + String databaseName, tableName; + databaseName = CatalogUtil.extractQualifier(table.getName()); + tableName = CatalogUtil.extractSimpleName(table.getName()); + Set<Predicate> predicates = new HashSet<>(); + for (EvalNode eval : PlannerUtil.getAllEqualEvals(qual)) { + BinaryEval binaryEval = (BinaryEval) eval; + // TODO: consider more complex predicates + if (binaryEval.getLeftExpr().getType() == EvalType.FIELD && + binaryEval.getRightExpr().getType() == EvalType.CONST) { + predicates.add(new Predicate(binaryEval.getType(), + ((FieldEval) binaryEval.getLeftExpr()).getColumnRef(), + ((ConstEval) binaryEval.getRightExpr()).getValue())); + } else if (binaryEval.getLeftExpr().getType() == EvalType.CONST && + binaryEval.getRightExpr().getType() == EvalType.FIELD) { + predicates.add(new Predicate(binaryEval.getType(), + ((FieldEval) binaryEval.getRightExpr()).getColumnRef(), + ((ConstEval) binaryEval.getLeftExpr()).getValue())); + } } - } - // for every subset of the set of columns, find all matched index paths - for (Set<Predicate> subset : Sets.powerSet(predicates)) { - if (subset.size() == 0) - continue; - Column[] columns = extractColumns(subset); - if (catalog.existIndexByColumns(databaseName, tableName, columns)) { - IndexDesc indexDesc = catalog.getIndexByColumns(databaseName, tableName, columns); - block.addAccessPath(scanNode, new IndexScanInfo( - table.getStats(), indexDesc, getSimplePredicates(indexDesc, subset))); + // for every subset of the set of columns, find all matched index paths + for (Set<Predicate> subset : Sets.powerSet(predicates)) { + if (subset.size() == 0) + continue; + Column[] columns = extractColumns(subset); + if (catalog.existIndexByColumns(databaseName, tableName, columns)) { + IndexDesc indexDesc = catalog.getIndexByColumns(databaseName, tableName, columns); + block.addAccessPath(scanNode, new IndexScanInfo( + table.getStats(), indexDesc, getSimplePredicates(indexDesc, subset))); + } } } } @@ -1023,7 +1028,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo private static SimplePredicate[] getSimplePredicates(IndexDesc desc, Set<Predicate> predicates) { SimplePredicate[] simplePredicates = new SimplePredicate[predicates.size()]; - Map<Column, Datum> colToValue = TUtil.newHashMap(); + Map<Column, Datum> colToValue = new HashMap<>(); for (Predicate predicate : predicates) { colToValue.put(predicate.column, predicate.value); }
