>From Ali Alsuliman <[email protected]>: Ali Alsuliman has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20997?usp=email )
Change subject: [ASTERIXDB-3713][COMP] Directive for extracting common ops ...................................................................... [ASTERIXDB-3713][COMP] Directive for extracting common ops - user model changes: no - storage format changes: no - interface changes: no Ext-ref: MB-70862 Change-Id: If6d8c832809715f2815f639950d5e0603e8e543b Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20997 Reviewed-by: Michael Blow <[email protected]> Tested-by: Ali Alsuliman <[email protected]> --- M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java M hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java 2 files changed, 90 insertions(+), 3 deletions(-) Approvals: Ali Alsuliman: Verified Michael Blow: Looks good to me, approved diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java index 3daf74b..98e26a2 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java @@ -43,6 +43,7 @@ import org.apache.asterix.optimizer.rules.util.EquivalenceClassUtils; import org.apache.asterix.translator.SqlppExpressionToPlanTranslator; import org.apache.asterix.translator.SqlppExpressionToPlanTranslatorFactory; +import org.apache.hyracks.algebricks.rewriter.rules.ExtractCommonOperatorsRule; public class SqlppCompilationProvider implements ILangCompilationProvider { @@ -107,6 +108,7 @@ DisjunctivePredicateToJoinRule.REWRITE_OR_AS_JOIN_OPTION, SetAsterixPhysicalOperatorsRule.REWRITE_ATTEMPT_BATCH_ASSIGN, EquivalenceClassUtils.REWRITE_INTERNAL_QUERYUID_PK, SqlppQueryRewriter.SQL_COMPAT_OPTION, - JoinEnum.CBO_FULL_ENUM_LEVEL_KEY, JoinEnum.CBO_CP_ENUM_KEY)); + JoinEnum.CBO_FULL_ENUM_LEVEL_KEY, JoinEnum.CBO_CP_ENUM_KEY, + ExtractCommonOperatorsRule.EXTRACT_COMMON_OPS)); } } diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java index 88cfaad..ae8a4f6 100644 --- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java +++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java @@ -60,6 +60,9 @@ */ public class ExtractCommonOperatorsRule implements IAlgebraicRewriteRule { + public static final String EXTRACT_COMMON_OPS = "extract_common_ops"; + public static final boolean EXTRACT_DEFAULT = true; + private final HashMap<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>> childrenToParents = new HashMap<>(); private final List<Mutable<ILogicalOperator>> roots = new ArrayList<>(); @@ -70,10 +73,15 @@ private int lastUsedClusterId = 0; private final Map<Mutable<ILogicalOperator>, BitSet> replicateToOutputs = new HashMap<>(); private final List<Pair<Mutable<ILogicalOperator>, Boolean>> newOutputs = new ArrayList<>(); + private Boolean extractCommonOpsEnabled = null; @Override public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { + if (extractCommonOpsEnabled == null) { + Object option = context.getMetadataProvider().getConfig().get(EXTRACT_COMMON_OPS); + extractCommonOpsEnabled = option != null ? Boolean.parseBoolean(option.toString()) : EXTRACT_DEFAULT; + } AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); if (op.getOperatorTag() != LogicalOperatorTag.WRITE && op.getOperatorTag() != LogicalOperatorTag.WRITE_RESULT && op.getOperatorTag() != LogicalOperatorTag.DISTRIBUTE_RESULT) { @@ -101,7 +109,11 @@ changed = false; // applying the rewriting until fixpoint topDownMaterialization(roots); - genCandidates(); + if (extractCommonOpsEnabled) { + genCandidatesGrow(); + } else { + genCandidates(); + } removeTrivialShare(); if (!equivalenceClasses.isEmpty()) { changed = rewrite(context); @@ -412,7 +424,7 @@ } } - private void genCandidates() throws AlgebricksException { + private void genCandidatesGrow() throws AlgebricksException { List<List<Mutable<ILogicalOperator>>> previousEquivalenceClasses = new ArrayList<>(); boolean grown = true; while (equivalenceClasses.size() > 0 && grown) { @@ -457,6 +469,40 @@ } } + private void genCandidates() throws AlgebricksException { + List<List<Mutable<ILogicalOperator>>> previousEquivalenceClasses = new ArrayList<>(); + while (equivalenceClasses.size() > 0) { + previousEquivalenceClasses.clear(); + for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) { + List<Mutable<ILogicalOperator>> candidatesCopy = new ArrayList<>(candidates); + previousEquivalenceClasses.add(candidatesCopy); + } + List<Mutable<ILogicalOperator>> currentLevelOpRefs = new ArrayList<>(); + for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) { + if (candidates.size() > 0) { + for (Mutable<ILogicalOperator> opRef : candidates) { + List<Mutable<ILogicalOperator>> refs = childrenToParents.get(opRef); + if (refs != null) { + currentLevelOpRefs.addAll(refs); + } + } + } + if (currentLevelOpRefs.size() == 0) { + continue; + } + candidates(currentLevelOpRefs, candidates); + } + if (currentLevelOpRefs.size() == 0) { + break; + } + prune(); + } + if (equivalenceClasses.size() < 1 && previousEquivalenceClasses.size() > 0) { + equivalenceClasses.addAll(previousEquivalenceClasses); + prune(); + } + } + private void topDownMaterialization(List<Mutable<ILogicalOperator>> tops) { List<Mutable<ILogicalOperator>> candidates = new ArrayList<>(); List<Mutable<ILogicalOperator>> nextLevel = new ArrayList<>(); @@ -526,6 +572,45 @@ return true; } + private boolean candidates(List<Mutable<ILogicalOperator>> opList, List<Mutable<ILogicalOperator>> candidates) { + List<Mutable<ILogicalOperator>> previousCandidates = new ArrayList<>(candidates); + candidates.clear(); + boolean validCandidate = false; + for (Mutable<ILogicalOperator> op : opList) { + List<Mutable<ILogicalOperator>> inputs = op.getValue().getInputs(); + for (int i = 0; i < inputs.size(); i++) { + Mutable<ILogicalOperator> inputRef = inputs.get(i); + validCandidate = false; + for (Mutable<ILogicalOperator> candidate : previousCandidates) { + // if current input is in candidates + if (inputRef.getValue().equals(candidate.getValue())) { + if (inputs.size() == 1) { + validCandidate = true; + } else { + BitSet candidateInputBitMap = opToCandidateInputs.get(op); + if (candidateInputBitMap == null) { + candidateInputBitMap = new BitSet(inputs.size()); + opToCandidateInputs.put(op, candidateInputBitMap); + } + candidateInputBitMap.set(i); + if (candidateInputBitMap.cardinality() == inputs.size()) { + validCandidate = true; + } + } + break; + } + } + } + if (!validCandidate) { + continue; + } + if (!candidates.contains(op)) { + candidates.add(op); + } + } + return true; + } + private void prune() throws AlgebricksException { List<List<Mutable<ILogicalOperator>>> previousEquivalenceClasses = new ArrayList<>(); for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) { -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20997?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: asterixdb Gerrit-Branch: phoenix Gerrit-Change-Id: If6d8c832809715f2815f639950d5e0603e8e543b Gerrit-Change-Number: 20997 Gerrit-PatchSet: 3 Gerrit-Owner: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]>
