Taewoo Kim has submitted this change and it was merged. Change subject: Added LeftOuterUnnestMap operator. ......................................................................
Added LeftOuterUnnestMap operator. - Added LeftOuterUnnestMap operator to represent the left-outer-join semantics properly. Change-Id: I4525899cf8e5e43551aa2ac2a78806ef6cc85e58 Reviewed-on: https://asterix-gerrit.ics.uci.edu/638 Tested-by: Jenkins <[email protected]> Reviewed-by: Yingyi Bu <[email protected]> --- M algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java A algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestMapOperator.java A algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/LeftOuterUnnestMapOperator.java M algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java M algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/CardinalityInferenceVisitor.java M algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java M algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java M algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java M algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalOperatorDeepCopyWithNewVariablesVisitor.java M algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java M algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java M algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java M algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java M algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java M algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java M algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java M algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java M algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java M algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/subplan/ReplaceNtsWithSubplanInputOperatorVisitor.java 19 files changed, 534 insertions(+), 200 deletions(-) Approvals: Yingyi Bu: Looks good to me, approved Jenkins: Verified diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java index 6361f42..cad24e1 100644 --- a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java @@ -33,6 +33,7 @@ INNERJOIN, INSERT_DELETE_UPSERT, LEFTOUTERJOIN, + LEFT_OUTER_UNNEST_MAP, LIMIT, MATERIALIZE, NESTEDTUPLESOURCE, diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestMapOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestMapOperator.java new file mode 100644 index 0000000..8a2981d --- /dev/null +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestMapOperator.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.algebricks.core.algebra.operators.logical; + +import java.util.List; + +import org.apache.commons.lang3.mutable.Mutable; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; +import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; +import org.apache.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy; + +public abstract class AbstractUnnestMapOperator extends AbstractUnnestOperator { + + protected final Mutable<ILogicalExpression> expression; + protected final List<Object> variableTypes; + protected boolean propagateInput; + protected List<Mutable<ILogicalExpression>> additionalFilteringExpressions; + protected List<LogicalVariable> minFilterVars; + protected List<LogicalVariable> maxFilterVars; + + public AbstractUnnestMapOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression, + List<Object> variableTypes, boolean propagateInput) { + super(variables, expression); + this.expression = expression; + this.variableTypes = variableTypes; + this.propagateInput = propagateInput; + } + + public List<Object> getVariableTypes() { + return variableTypes; + } + + /** + * If propagateInput is true, then propagates the input variables. + */ + @Override + public VariablePropagationPolicy getVariablePropagationPolicy() { + return new VariablePropagationPolicy() { + @Override + public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources) + throws AlgebricksException { + if (propagateInput) { + target.addAllVariables(sources[0]); + } + for (LogicalVariable v : variables) { + target.addVariable(v); + } + } + }; + } + + public boolean propagatesInput() { + return propagateInput; + } + + public void setPropagatesInput(boolean propagateInput) { + this.propagateInput = propagateInput; + } + + public List<LogicalVariable> getMinFilterVars() { + return minFilterVars; + } + + public void setMinFilterVars(List<LogicalVariable> minFilterVars) { + this.minFilterVars = minFilterVars; + } + + public List<LogicalVariable> getMaxFilterVars() { + return maxFilterVars; + } + + public void setMaxFilterVars(List<LogicalVariable> maxFilterVars) { + this.maxFilterVars = maxFilterVars; + } + + public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) { + this.additionalFilteringExpressions = additionalFilteringExpressions; + } + + public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() { + return additionalFilteringExpressions; + } + +} diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/LeftOuterUnnestMapOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/LeftOuterUnnestMapOperator.java new file mode 100644 index 0000000..56e2dfb --- /dev/null +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/LeftOuterUnnestMapOperator.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.algebricks.core.algebra.operators.logical; + +import java.util.List; + +import org.apache.commons.lang3.mutable.Mutable; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; +import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; +import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; +import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; +import org.apache.hyracks.algebricks.core.algebra.typing.ITypingContext; +import org.apache.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment; +import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor; + +/** + * Left-outer-unnest-map is similar to the unnest-map operator. The only + * difference is that this operator represents left-outer semantics, meaning + * that it generates null values for non-matching tuples. It also propagates all + * input variables. This may be used only in a left-outer-join case. + */ +public class LeftOuterUnnestMapOperator extends AbstractUnnestMapOperator { + + public LeftOuterUnnestMapOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression, + List<Object> variableTypes, boolean propagateInput) { + super(variables, expression, variableTypes, propagateInput); + // propagateInput is always set to true for this operator. + this.propagateInput = true; + } + + @Override + public LogicalOperatorTag getOperatorTag() { + return LogicalOperatorTag.LEFT_OUTER_UNNEST_MAP; + } + + @Override + public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException { + return visitor.visitLeftOuterUnnestMapOperator(this, arg); + } + + @Override + public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { + // Propagates all input variables that come from the outer branch. + PropagatingTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx); + + env.getCorrelatedNullableVariableLists().add(variables); + + // For the variables from the inner branch, the output type is the union + // of (original type + null). + for (int i = 0; i < variables.size(); i++) { + env.setVarType(variables.get(i), ctx.getNullableTypeComputer().makeNullableType(variableTypes.get(i))); + } + + return env; + } + +} \ No newline at end of file diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java index ebcc4f2..89e2423 100644 --- a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java @@ -26,23 +26,15 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; -import org.apache.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy; import org.apache.hyracks.algebricks.core.algebra.typing.ITypingContext; import org.apache.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment; import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor; -public class UnnestMapOperator extends AbstractUnnestOperator { - private final List<Object> variableTypes; - private boolean propagateInput; - private List<Mutable<ILogicalExpression>> additionalFilteringExpressions; - private List<LogicalVariable> minFilterVars; - private List<LogicalVariable> maxFilterVars; +public class UnnestMapOperator extends AbstractUnnestMapOperator { public UnnestMapOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression, List<Object> variableTypes, boolean propagateInput) { - super(variables, expression); - this.variableTypes = variableTypes; - this.propagateInput = propagateInput; + super(variables, expression, variableTypes, propagateInput); } @Override @@ -55,31 +47,8 @@ return visitor.visitUnnestMapOperator(this, arg); } - /** - * UnnestMap doesn't propagate input variables, because currently it is only - * used to search indexes. In the future, it would be nice to have the - * choice to propagate input variables or not. - */ - @Override - public VariablePropagationPolicy getVariablePropagationPolicy() { - return new VariablePropagationPolicy() { - @Override - public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources) - throws AlgebricksException { - if (propagateInput) { - target.addAllVariables(sources[0]); - } - for (LogicalVariable v : variables) { - target.addVariable(v); - } - } - }; - } - - public List<Object> getVariableTypes() { - return variableTypes; - } - + // When propagateInput is true, + // this operator propagates all input variables. @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { IVariableTypeEnvironment env = null; @@ -95,40 +64,4 @@ return env; } - public boolean propagatesInput() { - return propagateInput; - } - - public void setPropagatesInput(boolean propagateInput) { - this.propagateInput = propagateInput; - } - - public List<LogicalVariable> getMinFilterVars() { - return minFilterVars; - } - - public void setMinFilterVars(List<LogicalVariable> minFilterVars) { - this.minFilterVars = minFilterVars; - } - - public List<LogicalVariable> getMaxFilterVars() { - return maxFilterVars; - } - - public void setMaxFilterVars(List<LogicalVariable> maxFilterVars) { - this.maxFilterVars = maxFilterVars; - } - - public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) { - this.additionalFilteringExpressions = additionalFilteringExpressions; - } - - public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() { - return additionalFilteringExpressions; - } - - @Override - public boolean isMap() { - return true; - } } diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/CardinalityInferenceVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/CardinalityInferenceVisitor.java index 7b304ae..e90a685 100644 --- a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/CardinalityInferenceVisitor.java +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/CardinalityInferenceVisitor.java @@ -35,6 +35,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; @@ -57,11 +58,9 @@ import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor; /** - * A visitor that provides the basic inference of tuple cardinalities of an operator's - * output. - * There are only two cases: - * 1. the cardinality is one in the worst case; - * 2. the cardinality is some unknown value. + * A visitor that provides the basic inference of tuple cardinalities of an + * operator's output. There are only two cases: 1. the cardinality is one in the + * worst case; 2. the cardinality is some unknown value. */ public class CardinalityInferenceVisitor implements ILogicalOperatorVisitor<Long, Void> { private static final Long ONE = 1L; @@ -186,6 +185,11 @@ } @Override + public Long visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, Void arg) throws AlgebricksException { + return UNKNOWN; + } + + @Override public Long visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException { return UNKNOWN; } diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java index 1805289..9a6411b 100644 --- a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java @@ -60,6 +60,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; @@ -487,6 +488,28 @@ } @Override + public Void visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, IOptimizationContext ctx) + throws AlgebricksException { + // Unlike the unnest-map operator, we propagate all inputs since + // propagateInuput is always true. + Map<LogicalVariable, EquivalenceClass> equivalenceClasses = new HashMap<LogicalVariable, EquivalenceClass>(); + List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>(); + ctx.putEquivalenceClassMap(op, equivalenceClasses); + ctx.putFDList(op, functionalDependencies); + ILogicalOperator childOp = op.getInputs().get(0).getValue(); + functionalDependencies.addAll(getOrComputeFDs(childOp, ctx)); + equivalenceClasses.putAll(getOrComputeEqClasses(childOp, ctx)); + + // Like Left-Outer join case, we add functional dependencies. + List<LogicalVariable> leftSideVars = new ArrayList<LogicalVariable>(); + List<LogicalVariable> producedVars = new ArrayList<LogicalVariable>(); + VariableUtilities.getUsedVariables(op, leftSideVars); + VariableUtilities.getProducedVariables(op, leftSideVars); + functionalDependencies.add(new FunctionalDependency(leftSideVars, producedVars)); + return null; + } + + @Override public Void visitUnnestOperator(UnnestOperator op, IOptimizationContext ctx) throws AlgebricksException { fdsEqClassesForAbstractUnnestOperator(op, ctx); return null; @@ -579,13 +602,14 @@ } /*** - * Propagated equivalent classes from the child to the current operator, based - * on the used variables of the current operator. + * Propagated equivalent classes from the child to the current operator, + * based on the used variables of the current operator. * * @param op * , the current operator * @param ctx - * , the optimization context which keeps track of all equivalent classes. + * , the optimization context which keeps track of all equivalent + * classes. * @param usedVariables * , used variables. * @throws AlgebricksException @@ -627,9 +651,12 @@ } } - // Propagates equivalent classes that contain expressions that use the used variables. - // Note that for the case variable $v is not in the used variables but it is - // equivalent to field-access($t, i) and $t is a used variable, the equivalent + // Propagates equivalent classes that contain expressions that use the + // used variables. + // Note that for the case variable $v is not in the used variables but + // it is + // equivalent to field-access($t, i) and $t is a used variable, the + // equivalent // class should still be propagated (kept). Set<LogicalVariable> usedVarSet = new HashSet<LogicalVariable>(usedVariables); for (Entry<LogicalVariable, EquivalenceClass> entry : chldClasses.entrySet()) { @@ -642,7 +669,8 @@ if (!exprUsedVars.isEmpty()) { for (LogicalVariable v : ec.getMembers()) { eqClasses.put(v, ec); - // If variable members contain a used variable, the representative + // If variable members contain a used variable, the + // representative // variable should be a used variable. if (usedVarSet.contains(v)) { ec.setVariableRepresentative(v); @@ -676,8 +704,9 @@ Map<LogicalVariable, EquivalenceClass> eqClasses = getOrCreateEqClasses(op, ctx); Map<LogicalVariable, EquivalenceClass> propagatedEqClasses = getOrComputeEqClasses(inp1, ctx); /** - * The original eq classes of unnest-map are only for produced variables, therefore - * eqClasses and propagatedEqClasses do not have overlaps. + * The original eq classes of unnest-map are only for produced + * variables, therefore eqClasses and propagatedEqClasses do not have + * overlaps. */ eqClasses.putAll(propagatedEqClasses); ctx.putEquivalenceClassMap(op, eqClasses); @@ -756,11 +785,14 @@ Map<LogicalVariable, EquivalenceClass> newVarEqcMap = new HashMap<LogicalVariable, EquivalenceClass>(); for (Entry<LogicalVariable, EquivalenceClass> entry : eqClasses.entrySet()) { EquivalenceClass eqc = entry.getValue(); - // If the equivalence class contains the right-hand-side expression, - // the left-hand-side variable is added into the equivalence class. + // If the equivalence class contains the right-hand-side + // expression, + // the left-hand-side variable is added into the equivalence + // class. if (eqc.contains(expr)) { eqc.addMember(var); - newVarEqcMap.put(var, eqc); // Add var as a map key for the equivalence class. + newVarEqcMap.put(var, eqc); // Add var as a map key for the + // equivalence class. } } eqClasses.putAll(newVarEqcMap); diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java index f849a11..5ae116b 100644 --- a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java @@ -48,6 +48,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; @@ -83,8 +84,9 @@ @Override public Boolean visitAggregateOperator(AggregateOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.AGGREGATE) + if (aop.getOperatorTag() != LogicalOperatorTag.AGGREGATE) { return Boolean.FALSE; + } AggregateOperator aggOpArg = (AggregateOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = VariableUtilities.varListEqualUnordered( getPairList(op.getVariables(), op.getExpressions()), @@ -96,8 +98,9 @@ public Boolean visitRunningAggregateOperator(RunningAggregateOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.RUNNINGAGGREGATE) + if (aop.getOperatorTag() != LogicalOperatorTag.RUNNINGAGGREGATE) { return Boolean.FALSE; + } RunningAggregateOperator aggOpArg = (RunningAggregateOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = VariableUtilities.varListEqualUnordered( getPairList(op.getVariables(), op.getExpressions()), @@ -109,16 +112,18 @@ public Boolean visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) copyAndSubstituteVar(op, arg); - if (aop.getOperatorTag() != LogicalOperatorTag.EMPTYTUPLESOURCE) + if (aop.getOperatorTag() != LogicalOperatorTag.EMPTYTUPLESOURCE) { return Boolean.FALSE; + } return Boolean.TRUE; } @Override public Boolean visitExtensionOperator(ExtensionOperator op, ILogicalOperator arg) throws AlgebricksException { ExtensionOperator aop = (ExtensionOperator) copyAndSubstituteVar(op, arg); - if (aop.getOperatorTag() != LogicalOperatorTag.EXTENSION_OPERATOR) + if (aop.getOperatorTag() != LogicalOperatorTag.EXTENSION_OPERATOR) { return Boolean.FALSE; + } return Boolean.TRUE; } @@ -128,8 +133,9 @@ // require the same physical operator, otherwise delivers different data // properties if (aop.getOperatorTag() != LogicalOperatorTag.GROUP - || aop.getPhysicalOperator().getOperatorTag() != op.getPhysicalOperator().getOperatorTag()) + || aop.getPhysicalOperator().getOperatorTag() != op.getPhysicalOperator().getOperatorTag()) { return Boolean.FALSE; + } List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> keyLists = op.getGroupByList(); GroupByOperator gbyOpArg = (GroupByOperator) copyAndSubstituteVar(op, arg); @@ -145,12 +151,14 @@ boolean isomorphic = VariableUtilities.varListEqualUnordered(listLeft, listRight); - if (!isomorphic) + if (!isomorphic) { return Boolean.FALSE; + } int sizeOp = op.getNestedPlans().size(); int sizeArg = gbyOpArg.getNestedPlans().size(); - if (sizeOp != sizeArg) + if (sizeOp != sizeArg) { return Boolean.FALSE; + } GroupByOperator argOp = (GroupByOperator) arg; List<ILogicalPlan> plans = op.getNestedPlans(); @@ -158,14 +166,16 @@ for (int i = 0; i < plans.size(); i++) { List<Mutable<ILogicalOperator>> roots = plans.get(i).getRoots(); List<Mutable<ILogicalOperator>> rootsArg = plansArg.get(i).getRoots(); - if (roots.size() != rootsArg.size()) + if (roots.size() != rootsArg.size()) { return Boolean.FALSE; + } for (int j = 0; j < roots.size(); j++) { ILogicalOperator topOp1 = roots.get(j).getValue(); ILogicalOperator topOp2 = rootsArg.get(j).getValue(); isomorphic = IsomorphismUtilities.isOperatorIsomorphicPlanSegment(topOp1, topOp2); - if (!isomorphic) + if (!isomorphic) { return Boolean.FALSE; + } } } return isomorphic; @@ -174,11 +184,13 @@ @Override public Boolean visitLimitOperator(LimitOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.LIMIT) + if (aop.getOperatorTag() != LogicalOperatorTag.LIMIT) { return Boolean.FALSE; + } LimitOperator limitOpArg = (LimitOperator) copyAndSubstituteVar(op, arg); - if (op.getOffset() != limitOpArg.getOffset()) + if (op.getOffset() != limitOpArg.getOffset()) { return Boolean.FALSE; + } boolean isomorphic = op.getMaxObjects().getValue().equals(limitOpArg.getMaxObjects().getValue()); return isomorphic; } @@ -186,8 +198,9 @@ @Override public Boolean visitInnerJoinOperator(InnerJoinOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.INNERJOIN) + if (aop.getOperatorTag() != LogicalOperatorTag.INNERJOIN) { return Boolean.FALSE; + } InnerJoinOperator joinOpArg = (InnerJoinOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = op.getCondition().getValue().equals(joinOpArg.getCondition().getValue()); return isomorphic; @@ -197,8 +210,9 @@ public Boolean visitLeftOuterJoinOperator(LeftOuterJoinOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) + if (aop.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) { return Boolean.FALSE; + } LeftOuterJoinOperator joinOpArg = (LeftOuterJoinOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = op.getCondition().getValue().equals(joinOpArg.getCondition().getValue()); return isomorphic; @@ -208,16 +222,18 @@ public Boolean visitNestedTupleSourceOperator(NestedTupleSourceOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) + if (aop.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) { return Boolean.FALSE; + } return Boolean.TRUE; } @Override public Boolean visitOrderOperator(OrderOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.ORDER) + if (aop.getOperatorTag() != LogicalOperatorTag.ORDER) { return Boolean.FALSE; + } OrderOperator orderOpArg = (OrderOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = compareIOrderAndExpressions(op.getOrderExpressions(), orderOpArg.getOrderExpressions()); return isomorphic; @@ -226,8 +242,9 @@ @Override public Boolean visitAssignOperator(AssignOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.ASSIGN) + if (aop.getOperatorTag() != LogicalOperatorTag.ASSIGN) { return Boolean.FALSE; + } AssignOperator assignOpArg = (AssignOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = VariableUtilities.varListEqualUnordered( getPairList(op.getVariables(), op.getExpressions()), @@ -238,8 +255,9 @@ @Override public Boolean visitSelectOperator(SelectOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.SELECT) + if (aop.getOperatorTag() != LogicalOperatorTag.SELECT) { return Boolean.FALSE; + } SelectOperator selectOpArg = (SelectOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = op.getCondition().getValue().equals(selectOpArg.getCondition().getValue()); return isomorphic; @@ -248,8 +266,9 @@ @Override public Boolean visitProjectOperator(ProjectOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.PROJECT) + if (aop.getOperatorTag() != LogicalOperatorTag.PROJECT) { return Boolean.FALSE; + } ProjectOperator projectOpArg = (ProjectOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), projectOpArg.getVariables()); return isomorphic; @@ -259,8 +278,9 @@ public Boolean visitPartitioningSplitOperator(PartitioningSplitOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.PARTITIONINGSPLIT) + if (aop.getOperatorTag() != LogicalOperatorTag.PARTITIONINGSPLIT) { return Boolean.FALSE; + } PartitioningSplitOperator partitionOpArg = (PartitioningSplitOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = compareExpressions(op.getExpressions(), partitionOpArg.getExpressions()); return isomorphic; @@ -269,24 +289,27 @@ @Override public Boolean visitReplicateOperator(ReplicateOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.REPLICATE) + if (aop.getOperatorTag() != LogicalOperatorTag.REPLICATE) { return Boolean.FALSE; + } return Boolean.TRUE; } @Override public Boolean visitMaterializeOperator(MaterializeOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.MATERIALIZE) + if (aop.getOperatorTag() != LogicalOperatorTag.MATERIALIZE) { return Boolean.FALSE; + } return Boolean.TRUE; } @Override public Boolean visitScriptOperator(ScriptOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.SCRIPT) + if (aop.getOperatorTag() != LogicalOperatorTag.SCRIPT) { return Boolean.FALSE; + } ScriptOperator scriptOpArg = (ScriptOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = op.getScriptDescription().equals(scriptOpArg.getScriptDescription()); return isomorphic; @@ -295,22 +318,25 @@ @Override public Boolean visitSubplanOperator(SubplanOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.SUBPLAN) + if (aop.getOperatorTag() != LogicalOperatorTag.SUBPLAN) { return Boolean.FALSE; + } SubplanOperator subplanOpArg = (SubplanOperator) copyAndSubstituteVar(op, arg); List<ILogicalPlan> plans = op.getNestedPlans(); List<ILogicalPlan> plansArg = subplanOpArg.getNestedPlans(); for (int i = 0; i < plans.size(); i++) { List<Mutable<ILogicalOperator>> roots = plans.get(i).getRoots(); List<Mutable<ILogicalOperator>> rootsArg = plansArg.get(i).getRoots(); - if (roots.size() == rootsArg.size()) + if (roots.size() == rootsArg.size()) { return Boolean.FALSE; + } for (int j = 0; j < roots.size(); j++) { ILogicalOperator topOp1 = roots.get(j).getValue(); ILogicalOperator topOp2 = rootsArg.get(j).getValue(); boolean isomorphic = IsomorphismUtilities.isOperatorIsomorphicPlanSegment(topOp1, topOp2); - if (!isomorphic) + if (!isomorphic) { return Boolean.FALSE; + } } } return Boolean.TRUE; @@ -319,36 +345,38 @@ @Override public Boolean visitUnionOperator(UnionAllOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.UNIONALL) + if (aop.getOperatorTag() != LogicalOperatorTag.UNIONALL) { return Boolean.FALSE; + } UnionAllOperator unionOpArg = (UnionAllOperator) copyAndSubstituteVar(op, arg); List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> mapping = op.getVariableMappings(); List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> mappingArg = unionOpArg.getVariableMappings(); - if (mapping.size() != mappingArg.size()) + if (mapping.size() != mappingArg.size()) { return Boolean.FALSE; + } return VariableUtilities.varListEqualUnordered(mapping, mappingArg); } @Override public Boolean visitIntersectOperator(IntersectOperator op, ILogicalOperator arg) throws AlgebricksException { - if (op.getOperatorTag() != LogicalOperatorTag.INTERSECT){ + if (op.getOperatorTag() != LogicalOperatorTag.INTERSECT) { return Boolean.FALSE; } IntersectOperator intersetOpArg = (IntersectOperator) copyAndSubstituteVar(op, arg); List<LogicalVariable> variables = op.getOutputVars(); List<LogicalVariable> variablesArg = intersetOpArg.getOutputVars(); - if (variables.size() != variablesArg.size()){ + if (variables.size() != variablesArg.size()) { return Boolean.FALSE; } - if (!VariableUtilities.varListEqualUnordered(variables, variablesArg)){ + if (!VariableUtilities.varListEqualUnordered(variables, variablesArg)) { return Boolean.FALSE; } - if (op.getNumInput() != intersetOpArg.getNumInput()){ + if (op.getNumInput() != intersetOpArg.getNumInput()) { return Boolean.FALSE; } - for (int i = 0; i < op.getNumInput(); i++){ - if (!VariableUtilities.varListEqualUnordered(op.getInputVariables(i), intersetOpArg.getInputVariables(i))){ + for (int i = 0; i < op.getNumInput(); i++) { + if (!VariableUtilities.varListEqualUnordered(op.getInputVariables(i), intersetOpArg.getInputVariables(i))) { return Boolean.FALSE; } } @@ -358,13 +386,15 @@ @Override public Boolean visitUnnestOperator(UnnestOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.UNNEST) + if (aop.getOperatorTag() != LogicalOperatorTag.UNNEST) { return Boolean.FALSE; + } UnnestOperator unnestOpArg = (UnnestOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), unnestOpArg.getVariables()) && variableEqual(op.getPositionalVariable(), unnestOpArg.getPositionalVariable()); - if (!isomorphic) + if (!isomorphic) { return Boolean.FALSE; + } isomorphic = op.getExpressionRef().getValue().equals(unnestOpArg.getExpressionRef().getValue()); return isomorphic; } @@ -372,24 +402,44 @@ @Override public Boolean visitUnnestMapOperator(UnnestMapOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.UNNEST_MAP) + if (aop.getOperatorTag() != LogicalOperatorTag.UNNEST_MAP) { return Boolean.FALSE; + } UnnestMapOperator unnestOpArg = (UnnestMapOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), unnestOpArg.getVariables()); - if (!isomorphic) + if (!isomorphic) { return Boolean.FALSE; + } isomorphic = op.getExpressionRef().getValue().equals(unnestOpArg.getExpressionRef().getValue()); + return isomorphic; + } + + @Override + public Boolean visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, ILogicalOperator arg) + throws AlgebricksException { + AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; + if (aop.getOperatorTag() != LogicalOperatorTag.LEFT_OUTER_UNNEST_MAP) { + return Boolean.FALSE; + } + LeftOuterUnnestMapOperator loUnnestOpArg = (LeftOuterUnnestMapOperator) copyAndSubstituteVar(op, arg); + boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), loUnnestOpArg.getVariables()); + if (!isomorphic) { + return Boolean.FALSE; + } + isomorphic = op.getExpressionRef().getValue().equals(loUnnestOpArg.getExpressionRef().getValue()); return isomorphic; } @Override public Boolean visitDataScanOperator(DataSourceScanOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) + if (aop.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) { return Boolean.FALSE; + } DataSourceScanOperator argScan = (DataSourceScanOperator) arg; - if (!argScan.getDataSource().toString().equals(op.getDataSource().toString())) + if (!argScan.getDataSource().toString().equals(op.getDataSource().toString())) { return Boolean.FALSE; + } DataSourceScanOperator scanOpArg = (DataSourceScanOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), scanOpArg.getVariables()) && op.getDataSource().toString().equals(scanOpArg.getDataSource().toString()); @@ -399,8 +449,9 @@ @Override public Boolean visitDistinctOperator(DistinctOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.DISTINCT) + if (aop.getOperatorTag() != LogicalOperatorTag.DISTINCT) { return Boolean.FALSE; + } DistinctOperator distinctOpArg = (DistinctOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = compareExpressions(op.getExpressions(), distinctOpArg.getExpressions()); return isomorphic; @@ -409,36 +460,44 @@ @Override public Boolean visitExchangeOperator(ExchangeOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.EXCHANGE) + if (aop.getOperatorTag() != LogicalOperatorTag.EXCHANGE) { return Boolean.FALSE; + } // require the same partition property - if (!(op.getPhysicalOperator().getOperatorTag() == aop.getPhysicalOperator().getOperatorTag())) + if (!(op.getPhysicalOperator().getOperatorTag() == aop.getPhysicalOperator().getOperatorTag())) { return Boolean.FALSE; + } variableMapping.clear(); IsomorphismUtilities.mapVariablesTopDown(op, arg, variableMapping); IPhysicalPropertiesVector properties = op.getPhysicalOperator().getDeliveredProperties(); IPhysicalPropertiesVector propertiesArg = aop.getPhysicalOperator().getDeliveredProperties(); - if (properties == null && propertiesArg == null) + if (properties == null && propertiesArg == null) { return Boolean.TRUE; - if (properties == null || propertiesArg == null) + } + if (properties == null || propertiesArg == null) { return Boolean.FALSE; + } IPartitioningProperty partProp = properties.getPartitioningProperty(); IPartitioningProperty partPropArg = propertiesArg.getPartitioningProperty(); - if (!partProp.getPartitioningType().equals(partPropArg.getPartitioningType())) + if (!partProp.getPartitioningType().equals(partPropArg.getPartitioningType())) { return Boolean.FALSE; + } List<LogicalVariable> columns = new ArrayList<LogicalVariable>(); partProp.getColumns(columns); List<LogicalVariable> columnsArg = new ArrayList<LogicalVariable>(); partPropArg.getColumns(columnsArg); - if (columns.size() != columnsArg.size()) + if (columns.size() != columnsArg.size()) { return Boolean.FALSE; - if (columns.size() == 0) + } + if (columns.size() == 0) { return Boolean.TRUE; + } for (int i = 0; i < columnsArg.size(); i++) { LogicalVariable rightVar = columnsArg.get(i); LogicalVariable leftVar = variableMapping.get(rightVar); - if (leftVar != null) + if (leftVar != null) { columnsArg.set(i, leftVar); + } } return VariableUtilities.varListEqualUnordered(columns, columnsArg); } @@ -446,8 +505,9 @@ @Override public Boolean visitWriteOperator(WriteOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.WRITE) + if (aop.getOperatorTag() != LogicalOperatorTag.WRITE) { return Boolean.FALSE; + } WriteOperator writeOpArg = (WriteOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), writeOpArg.getSchema()); return isomorphic; @@ -457,8 +517,9 @@ public Boolean visitDistributeResultOperator(DistributeResultOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.DISTRIBUTE_RESULT) + if (aop.getOperatorTag() != LogicalOperatorTag.DISTRIBUTE_RESULT) { return Boolean.FALSE; + } DistributeResultOperator writeOpArg = (DistributeResultOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), writeOpArg.getSchema()); return isomorphic; @@ -467,28 +528,35 @@ @Override public Boolean visitWriteResultOperator(WriteResultOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.WRITE_RESULT) + if (aop.getOperatorTag() != LogicalOperatorTag.WRITE_RESULT) { return Boolean.FALSE; + } WriteResultOperator writeOpArg = (WriteResultOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), writeOpArg.getSchema()); - if (!op.getDataSource().equals(writeOpArg.getDataSource())) + if (!op.getDataSource().equals(writeOpArg.getDataSource())) { isomorphic = false; - if (!op.getPayloadExpression().equals(writeOpArg.getPayloadExpression())) + } + if (!op.getPayloadExpression().equals(writeOpArg.getPayloadExpression())) { isomorphic = false; + } return isomorphic; } @Override - public Boolean visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, ILogicalOperator arg) throws AlgebricksException { + public Boolean visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, ILogicalOperator arg) + throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.INSERT_DELETE_UPSERT) + if (aop.getOperatorTag() != LogicalOperatorTag.INSERT_DELETE_UPSERT) { return Boolean.FALSE; + } InsertDeleteUpsertOperator insertOpArg = (InsertDeleteUpsertOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), insertOpArg.getSchema()); - if (!op.getDataSource().equals(insertOpArg.getDataSource())) + if (!op.getDataSource().equals(insertOpArg.getDataSource())) { isomorphic = false; - if (!op.getPayloadExpression().equals(insertOpArg.getPayloadExpression())) + } + if (!op.getPayloadExpression().equals(insertOpArg.getPayloadExpression())) { isomorphic = false; + } return isomorphic; } @@ -496,24 +564,28 @@ public Boolean visitIndexInsertDeleteUpsertOperator(IndexInsertDeleteUpsertOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.INDEX_INSERT_DELETE_UPSERT) + if (aop.getOperatorTag() != LogicalOperatorTag.INDEX_INSERT_DELETE_UPSERT) { return Boolean.FALSE; + } IndexInsertDeleteUpsertOperator insertOpArg = (IndexInsertDeleteUpsertOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), insertOpArg.getSchema()); - if (!op.getDataSourceIndex().equals(insertOpArg.getDataSourceIndex())) + if (!op.getDataSourceIndex().equals(insertOpArg.getDataSourceIndex())) { isomorphic = false; + } return isomorphic; } @Override public Boolean visitTokenizeOperator(TokenizeOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.TOKENIZE) + if (aop.getOperatorTag() != LogicalOperatorTag.TOKENIZE) { return Boolean.FALSE; + } TokenizeOperator tokenizeOpArg = (TokenizeOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), tokenizeOpArg.getSchema()); - if (!op.getDataSourceIndex().equals(tokenizeOpArg.getDataSourceIndex())) + if (!op.getDataSourceIndex().equals(tokenizeOpArg.getDataSourceIndex())) { isomorphic = false; + } return isomorphic; } @@ -524,27 +596,32 @@ private Boolean compareExpressions(List<Mutable<ILogicalExpression>> opExprs, List<Mutable<ILogicalExpression>> argExprs) { - if (opExprs.size() != argExprs.size()) + if (opExprs.size() != argExprs.size()) { return Boolean.FALSE; + } for (int i = 0; i < opExprs.size(); i++) { boolean isomorphic = opExprs.get(i).getValue().equals(argExprs.get(i).getValue()); - if (!isomorphic) + if (!isomorphic) { return Boolean.FALSE; + } } return Boolean.TRUE; } private Boolean compareIOrderAndExpressions(List<Pair<IOrder, Mutable<ILogicalExpression>>> opOrderExprs, List<Pair<IOrder, Mutable<ILogicalExpression>>> argOrderExprs) { - if (opOrderExprs.size() != argOrderExprs.size()) + if (opOrderExprs.size() != argOrderExprs.size()) { return Boolean.FALSE; + } for (int i = 0; i < opOrderExprs.size(); i++) { boolean isomorphic = opOrderExprs.get(i).first.equals(argOrderExprs.get(i).first); - if (!isomorphic) + if (!isomorphic) { return Boolean.FALSE; + } isomorphic = opOrderExprs.get(i).second.getValue().equals(argOrderExprs.get(i).second.getValue()); - if (!isomorphic) + if (!isomorphic) { return Boolean.FALSE; + } } return Boolean.TRUE; } @@ -556,16 +633,18 @@ IsomorphismUtilities.mapVariablesTopDown(op, argOp, variableMapping); List<LogicalVariable> liveVars = new ArrayList<LogicalVariable>(); - if (argOp.getInputs().size() > 0) + if (argOp.getInputs().size() > 0) { for (int i = 0; i < argOp.getInputs().size(); i++) VariableUtilities.getLiveVariables(argOp.getInputs().get(i).getValue(), liveVars); + } List<LogicalVariable> producedVars = new ArrayList<LogicalVariable>(); VariableUtilities.getProducedVariables(argOp, producedVars); List<LogicalVariable> producedVarsNew = new ArrayList<LogicalVariable>(); VariableUtilities.getProducedVariables(op, producedVarsNew); - if (producedVars.size() != producedVarsNew.size()) + if (producedVars.size() != producedVarsNew.size()) { return newOp; + } for (Entry<LogicalVariable, LogicalVariable> map : variableMapping.entrySet()) { if (liveVars.contains(map.getKey())) { VariableUtilities.substituteVariables(newOp, map.getKey(), map.getValue(), null); @@ -579,8 +658,9 @@ public List<Pair<LogicalVariable, ILogicalExpression>> getPairList(List<LogicalVariable> vars, List<Mutable<ILogicalExpression>> exprs) throws AlgebricksException { List<Pair<LogicalVariable, ILogicalExpression>> list = new ArrayList<Pair<LogicalVariable, ILogicalExpression>>(); - if (vars.size() != exprs.size()) + if (vars.size() != exprs.size()) { throw new AlgebricksException("variable list size does not equal to expression list size "); + } for (int i = 0; i < vars.size(); i++) { list.add(new Pair<LogicalVariable, ILogicalExpression>(vars.get(i), exprs.get(i).getValue())); } @@ -588,24 +668,28 @@ } private static boolean variableEqual(LogicalVariable var, LogicalVariable varArg) { - if (var == null && varArg == null) + if (var == null && varArg == null) { return true; - if (var.equals(varArg)) + } + if (var.equals(varArg)) { return true; - else + } else { return false; + } } @Override public Boolean visitOuterUnnestOperator(OuterUnnestOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; - if (aop.getOperatorTag() != LogicalOperatorTag.OUTER_UNNEST) + if (aop.getOperatorTag() != LogicalOperatorTag.OUTER_UNNEST) { return Boolean.FALSE; + } OuterUnnestOperator unnestOpArg = (OuterUnnestOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), unnestOpArg.getVariables()) && variableEqual(op.getPositionalVariable(), unnestOpArg.getPositionalVariable()); - if (!isomorphic) + if (!isomorphic) { return Boolean.FALSE; + } isomorphic = op.getExpressionRef().getValue().equals(unnestOpArg.getExpressionRef().getValue()); return isomorphic; } diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java index b81f747..1b1b9d8 100644 --- a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java @@ -49,6 +49,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; @@ -220,6 +221,13 @@ } @Override + public Void visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, ILogicalOperator arg) + throws AlgebricksException { + mapVariablesStandard(op, arg); + return null; + } + + @Override public Void visitDataScanOperator(DataSourceScanOperator op, ILogicalOperator arg) throws AlgebricksException { mapVariablesStandard(op, arg); return null; @@ -257,7 +265,8 @@ } @Override - public Void visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, ILogicalOperator arg) throws AlgebricksException { + public Void visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, ILogicalOperator arg) + throws AlgebricksException { mapVariablesStandard(op, arg); return null; } @@ -435,12 +444,12 @@ private void mapVariablesForIntersect(IntersectOperator op, ILogicalOperator arg) { IntersectOperator opArg = (IntersectOperator) arg; - if (op.getNumInput() != opArg.getNumInput()){ + if (op.getNumInput() != opArg.getNumInput()) { return; } - for (int i = 0; i < op.getNumInput(); i++){ - for (int j = 0; j < op.getInputVariables(i).size(); j++){ - if (!varEquivalent(op.getInputVariables(i).get(j), opArg.getInputVariables(i).get(j))){ + for (int i = 0; i < op.getNumInput(); i++) { + for (int j = 0; j < op.getInputVariables(i).size(); j++) { + if (!varEquivalent(op.getInputVariables(i).get(j), opArg.getInputVariables(i).get(j))) { return; } } diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalOperatorDeepCopyWithNewVariablesVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalOperatorDeepCopyWithNewVariablesVisitor.java index 3905d13..213b2b1 100644 --- a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalOperatorDeepCopyWithNewVariablesVisitor.java +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalOperatorDeepCopyWithNewVariablesVisitor.java @@ -46,6 +46,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; @@ -70,9 +71,9 @@ import org.apache.hyracks.algebricks.core.algebra.visitors.IQueryOperatorVisitor; /** - * This visitor deep-copies a query plan but uses a new set of variables. - * Method getInputToOutputVariableMapping() will return a map that maps - * input variables to their corresponding output variables. + * This visitor deep-copies a query plan but uses a new set of variables. Method + * getInputToOutputVariableMapping() will return a map that maps input variables + * to their corresponding output variables. */ public class LogicalOperatorDeepCopyWithNewVariablesVisitor implements IQueryOperatorVisitor<ILogicalOperator, ILogicalOperator> { @@ -84,12 +85,13 @@ // original one in the copied plan. private final Map<LogicalVariable, LogicalVariable> inputVarToOutputVarMapping; - // Key: New variable in the new plan. Value: The old variable in the original plan. + // Key: New variable in the new plan. Value: The old variable in the + // original plan. private final Map<LogicalVariable, LogicalVariable> outputVarToInputVarMapping; /** - * @param IOptimizationContext, - * the optimization context + * @param IOptimizationContext + * , the optimization context */ public LogicalOperatorDeepCopyWithNewVariablesVisitor(IVariableContext varContext, ITypingContext typeContext) { this.varContext = varContext; @@ -478,7 +480,7 @@ throws AlgebricksException { List<List<LogicalVariable>> liveVarsInInputs = getLiveVarsInInputs(op); List<LogicalVariable> outputCopy = new ArrayList<>(); - for (LogicalVariable var : op.getOutputVars()){ + for (LogicalVariable var : op.getOutputVars()) { outputCopy.add(deepCopyVariable(var)); } IntersectOperator opCopy = new IntersectOperator(outputCopy, liveVarsInInputs); @@ -511,6 +513,16 @@ } @Override + public ILogicalOperator visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, ILogicalOperator arg) + throws AlgebricksException { + LeftOuterUnnestMapOperator opCopy = new LeftOuterUnnestMapOperator(deepCopyVariableList(op.getVariables()), + exprDeepCopyVisitor.deepCopyExpressionReference(op.getExpressionRef()), op.getVariableTypes(), + op.propagatesInput()); + deepCopyInputsAnnotationsAndExecutionMode(op, arg, opCopy); + return opCopy; + } + + @Override public ILogicalOperator visitUnnestOperator(UnnestOperator op, ILogicalOperator arg) throws AlgebricksException { UnnestOperator opCopy = new UnnestOperator(deepCopyVariable(op.getVariable()), exprDeepCopyVisitor.deepCopyExpressionReference(op.getExpressionRef()), diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java index 2f3725d..7ef76ee 100644 --- a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java @@ -38,9 +38,10 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; @@ -240,6 +241,12 @@ } @Override + public Void visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, IOptimizationContext arg) + throws AlgebricksException { + return null; + } + + @Override public Void visitUnnestOperator(UnnestOperator op, IOptimizationContext arg) throws AlgebricksException { // TODO Auto-generated method stub return null; diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java index 7fd3c46..931640e 100644 --- a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java @@ -43,9 +43,10 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; @@ -209,7 +210,7 @@ public ILogicalOperator visitIntersectOperator(IntersectOperator op, Void arg) throws AlgebricksException { List<LogicalVariable> outputVar = new ArrayList<>(op.getOutputVars()); List<List<LogicalVariable>> inputVars = new ArrayList<>(op.getNumInput()); - for(int i = 0; i < op.getNumInput(); i++){ + for (int i = 0; i < op.getNumInput(); i++) { inputVars.add(new ArrayList<>(op.getInputVariables(i))); } return new IntersectOperator(outputVar, inputVars); @@ -226,6 +227,15 @@ ArrayList<LogicalVariable> newInputList = new ArrayList<LogicalVariable>(); newInputList.addAll(op.getVariables()); return new UnnestMapOperator(newInputList, deepCopyExpressionRef(op.getExpressionRef()), + new ArrayList<Object>(op.getVariableTypes()), op.propagatesInput()); + } + + @Override + public ILogicalOperator visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, Void arg) + throws AlgebricksException { + ArrayList<LogicalVariable> newInputList = new ArrayList<LogicalVariable>(); + newInputList.addAll(op.getVariables()); + return new LeftOuterUnnestMapOperator(newInputList, deepCopyExpressionRef(op.getExpressionRef()), new ArrayList<Object>(op.getVariableTypes()), op.propagatesInput()); } @@ -276,7 +286,8 @@ } @Override - public ILogicalOperator visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, Void arg) throws AlgebricksException { + public ILogicalOperator visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, Void arg) + throws AlgebricksException { List<Mutable<ILogicalExpression>> newKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>(); deepCopyExpressionRefs(newKeyExpressions, op.getPrimaryKeyExpressions()); List<Mutable<ILogicalExpression>> newLSMComponentFilterExpressions = new ArrayList<Mutable<ILogicalExpression>>(); @@ -299,9 +310,9 @@ ((AbstractLogicalExpression) op.getFilterExpression()).cloneExpression()); List<Mutable<ILogicalExpression>> newLSMComponentFilterExpressions = new ArrayList<Mutable<ILogicalExpression>>(); deepCopyExpressionRefs(newLSMComponentFilterExpressions, op.getAdditionalFilteringExpressions()); - IndexInsertDeleteUpsertOperator indexInsertDeleteOp = new IndexInsertDeleteUpsertOperator(op.getDataSourceIndex(), - newPrimaryKeyExpressions, newSecondaryKeyExpressions, newFilterExpression, op.getOperation(), - op.isBulkload()); + IndexInsertDeleteUpsertOperator indexInsertDeleteOp = new IndexInsertDeleteUpsertOperator( + op.getDataSourceIndex(), newPrimaryKeyExpressions, newSecondaryKeyExpressions, newFilterExpression, + op.getOperation(), op.isBulkload()); indexInsertDeleteOp.setAdditionalFilteringExpressions(newLSMComponentFilterExpressions); return indexInsertDeleteOp; } diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java index 8a0f156..b05bdf5 100644 --- a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java @@ -47,6 +47,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; @@ -223,6 +224,12 @@ } @Override + public Void visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, Void arg) throws AlgebricksException { + producedVariables.addAll(op.getVariables()); + return null; + } + + @Override public Void visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException { return visitUnnestNonMapOperator(op); } diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java index 7a2f229..ea6f6aa 100644 --- a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java @@ -45,6 +45,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; @@ -244,6 +245,12 @@ } @Override + public Void visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, Void arg) throws AlgebricksException { + standardLayout(op); + return null; + } + + @Override public Void visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException { standardLayout(op); return null; diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java index 55dc11a..4061497 100644 --- a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java @@ -29,6 +29,7 @@ import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan; import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestNonMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator; @@ -41,9 +42,10 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; @@ -306,13 +308,13 @@ public Void visitIntersectOperator(IntersectOperator op, Pair<LogicalVariable, LogicalVariable> pair) throws AlgebricksException { for (int i = 0; i < op.getOutputVars().size(); i++) { - if (op.getOutputVars().get(i).equals(pair.first)){ + if (op.getOutputVars().get(i).equals(pair.first)) { op.getOutputVars().set(i, pair.second); } } - for(int i = 0; i < op.getNumInput(); i++){ - for (int j = 0; j < op.getInputVariables(i).size(); j++){ - if (op.getInputVariables(i).get(j).equals(pair.first)){ + for (int i = 0; i < op.getNumInput(); i++) { + for (int j = 0; j < op.getInputVariables(i).size(); j++) { + if (op.getInputVariables(i).get(j).equals(pair.first)) { op.getInputVariables(i).set(j, pair.second); } } @@ -323,16 +325,28 @@ @Override public Void visitUnnestMapOperator(UnnestMapOperator op, Pair<LogicalVariable, LogicalVariable> pair) throws AlgebricksException { + substituteVarsForAbstractUnnestMapOp(op, pair); + return null; + } + + @Override + public Void visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, + Pair<LogicalVariable, LogicalVariable> pair) throws AlgebricksException { + substituteVarsForAbstractUnnestMapOp(op, pair); + return null; + } + + private void substituteVarsForAbstractUnnestMapOp(AbstractUnnestMapOperator op, + Pair<LogicalVariable, LogicalVariable> pair) throws AlgebricksException { List<LogicalVariable> variables = op.getVariables(); for (int i = 0; i < variables.size(); i++) { if (variables.get(i) == pair.first) { variables.set(i, pair.second); - return null; + return; } } op.getExpressionRef().getValue().substituteVar(pair.first, pair.second); substVarTypes(op, pair); - return null; } @Override diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java index 5cfd619..6e895bd 100644 --- a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java @@ -30,6 +30,7 @@ import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan; import org.apache.hyracks.algebricks.core.algebra.base.IPhysicalOperator; import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator; @@ -44,6 +45,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; @@ -310,13 +312,23 @@ @Override public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) { + getUsedVarsFromExprAndFilterExpr(op); + return null; + } + + @Override + public Void visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, Void arg) throws AlgebricksException { + getUsedVarsFromExprAndFilterExpr(op); + return null; + } + + private void getUsedVarsFromExprAndFilterExpr(AbstractUnnestMapOperator op) { op.getExpressionRef().getValue().getUsedVariables(usedVariables); if (op.getAdditionalFilteringExpressions() != null) { for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) { e.getValue().getUsedVariables(usedVariables); } } - return null; } @Override @@ -357,14 +369,14 @@ @Override public Void visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, Void arg) { - //1. The record variable + // 1. The record variable op.getPayloadExpression().getValue().getUsedVariables(usedVariables); - //2. The primary key variables + // 2. The primary key variables for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) { e.getValue().getUsedVariables(usedVariables); } - //3. The filters variables + // 3. The filters variables if (op.getAdditionalFilteringExpressions() != null) { for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) { e.getValue().getUsedVariables(usedVariables); diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java index ca35284..b85202d 100644 --- a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java @@ -28,6 +28,7 @@ import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan; import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator; @@ -43,6 +44,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator.Kind; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; @@ -306,8 +308,19 @@ @Override public String visitUnnestMapOperator(UnnestMapOperator op, Integer indent) throws AlgebricksException { + return printAbstractUnnestMapOperator(op, indent, "unnest-map"); + } + + @Override + public String visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, Integer indent) + throws AlgebricksException { + return printAbstractUnnestMapOperator(op, indent, "left-outer-unnest-map"); + } + + private String printAbstractUnnestMapOperator(AbstractUnnestMapOperator op, Integer indent, String opSignature) + throws AlgebricksException { StringBuilder buffer = new StringBuilder(); - addIndent(buffer, indent).append("unnest-map " + op.getVariables() + " <- " + addIndent(buffer, indent).append(opSignature + " " + op.getVariables() + " <- " + op.getExpressionRef().getValue().accept(exprVisitor, indent)); return buffer.toString(); } diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java index fa9ad05..e011a7f 100644 --- a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java @@ -70,9 +70,9 @@ Object t = typeEnvs[i].getTypeEnv().getVarType(var, nonNullVariableList, correlatedNullableVariableLists); if (t != null) { - if (i == 0) { // inner branch + if (i == 0) { // outer branch return t; - } else { // outer branch + } else { // inner branch boolean nonNullVarIsProduced = false; for (LogicalVariable v : nonNullVariableList) { if (v == var) { diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java index ebb0198..298daf9 100644 --- a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java +++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java @@ -33,6 +33,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; @@ -103,6 +104,8 @@ public R visitUnnestMapOperator(UnnestMapOperator op, T arg) throws AlgebricksException; + public R visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, T arg) throws AlgebricksException; + public R visitDataScanOperator(DataSourceScanOperator op, T arg) throws AlgebricksException; public R visitDistinctOperator(DistinctOperator op, T arg) throws AlgebricksException; diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/subplan/ReplaceNtsWithSubplanInputOperatorVisitor.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/subplan/ReplaceNtsWithSubplanInputOperatorVisitor.java index 551b5d0..d36bff9 100644 --- a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/subplan/ReplaceNtsWithSubplanInputOperatorVisitor.java +++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/subplan/ReplaceNtsWithSubplanInputOperatorVisitor.java @@ -41,6 +41,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; @@ -62,9 +63,10 @@ import org.apache.hyracks.algebricks.core.algebra.visitors.IQueryOperatorVisitor; /** - * This visitor replaces NTS' in a subplan with its input operator or its deep copies. - * Note that this visitor can only be used in the rule EliminateSubplanWithInputCardinalityOneRule, - * for cases where the Subplan input operator is of cardinality one and its variables are not needed after + * This visitor replaces NTS' in a subplan with its input operator or its deep + * copies. Note that this visitor can only be used in the rule + * EliminateSubplanWithInputCardinalityOneRule, for cases where the Subplan + * input operator is of cardinality one and its variables are not needed after * the Subplan. */ class ReplaceNtsWithSubplanInputOperatorVisitor implements IQueryOperatorVisitor<ILogicalOperator, Void> { @@ -74,7 +76,8 @@ // The input operator to the subplan. private final ILogicalOperator subplanInputOperator; - // The map that maps the input variables to the subplan to their deep-copied variables. + // The map that maps the input variables to the subplan to their deep-copied + // variables. private final Map<LogicalVariable, LogicalVariable> varMap = new HashMap<>(); // Whether the original copy has been used. @@ -216,6 +219,12 @@ } @Override + public ILogicalOperator visitLeftOuterUnnestMapOperator(LeftOuterUnnestMapOperator op, Void arg) + throws AlgebricksException { + return visit(op); + } + + @Override public ILogicalOperator visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException { return visit(op); } @@ -245,7 +254,8 @@ for (Mutable<ILogicalOperator> childRef : op.getInputs()) { ILogicalOperator newChild = childRef.getValue().accept(this, null); childRef.setValue(newChild); - // Replaces variables in op with the mapping obtained from one child. + // Replaces variables in op with the mapping obtained from one + // child. VariableUtilities.substituteVariables(op, varMap, ctx); // Keep the map from current child and move to the next child. varMapSnapshots.add(new HashMap<LogicalVariable, LogicalVariable>(varMap)); -- To view, visit https://asterix-gerrit.ics.uci.edu/638 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I4525899cf8e5e43551aa2ac2a78806ef6cc85e58 Gerrit-PatchSet: 12 Gerrit-Project: hyracks Gerrit-Branch: master Gerrit-Owner: Taewoo Kim <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Taewoo Kim <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: Yingyi Bu <[email protected]> Gerrit-Reviewer: Young-Seok Kim <[email protected]>
