This is an automated email from the ASF dual-hosted git repository.
dlych pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 3c71d2b [NO ISSUE][COMP] Improve
IPartitioningProperty.substituteColumnVars()
3c71d2b is described below
commit 3c71d2bd02a68cf3eedfab1491d7bbe6911649bc
Author: Dmitry Lychagin <[email protected]>
AuthorDate: Wed Jul 17 14:19:19 2019 -0700
[NO ISSUE][COMP] Improve IPartitioningProperty.substituteColumnVars()
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- IPartitioningProperty.substituteColumnVars() should return
a new instance of IPartitioningProperty if variable
substitution was performed for given variables
- Compute schema for a new Select operator introduced by
PullSelectOutOfEqJoin rule
- Refactor delivered properties computation in
AbstractPreclusteredGroupByPOperator
Change-Id: Iee7ef7de26f9d960b205d6d43e2820aaa396cb15
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3499
Contrib: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Integration-Tests: Jenkins <[email protected]>
Reviewed-by: Dmitry Lychagin <[email protected]>
Reviewed-by: Ali Alsuliman <[email protected]>
---
.../AbstractPreclusteredGroupByPOperator.java | 50 +++++++++++-----------
.../operators/physical/IntersectPOperator.java | 4 +-
.../properties/BroadcastPartitioningProperty.java | 8 +++-
.../algebra/properties/IPartitioningProperty.java | 7 +--
.../properties/OrderedPartitionedProperty.java | 17 +++++---
.../properties/RandomPartitioningProperty.java | 4 +-
.../properties/UnorderedPartitionedProperty.java | 16 ++++---
.../rewriter/rules/PullSelectOutOfEqJoin.java | 1 +
8 files changed, 63 insertions(+), 44 deletions(-)
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
index fdb7347..70f7e2b 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
@@ -63,35 +63,37 @@ public abstract class AbstractPreclusteredGroupByPOperator
extends AbstractGroup
super(columnList);
}
- // Obs: We don't propagate properties corresponding to decors, since they
- // are func. dep. on the group-by variables.
+ // Obs: We don't propagate properties corresponding to decors, since they
are func. dep. on the group-by variables.
@Override
public void computeDeliveredProperties(ILogicalOperator op,
IOptimizationContext context) {
- List<ILocalStructuralProperty> propsLocal = new ArrayList<>();
GroupByOperator gby = (GroupByOperator) op;
- ILogicalOperator op2 = gby.getInputs().get(0).getValue();
- IPhysicalPropertiesVector childProp =
op2.getDeliveredPhysicalProperties();
- IPartitioningProperty pp = childProp.getPartitioningProperty();
- Map<LogicalVariable, LogicalVariable> ppSubstMap =
computePartitioningPropertySubstitutionMap(gby, pp);
- if (ppSubstMap != null) {
- // We cannot modify pp directly, since it is owned by the input
operator.
- // Otherwise, the partitioning property would be modified even
before this group by operator,
- // which will be undesirable.
- pp = pp.clonePartitioningProperty();
- pp.substituteColumnVars(ppSubstMap);
- }
- List<ILocalStructuralProperty> childLocals =
childProp.getLocalProperties();
- if (childLocals == null) {
- deliveredProperties = new StructuralPropertiesVector(pp,
propsLocal);
- return;
- }
- for (ILocalStructuralProperty lsp : childLocals) {
- ILocalStructuralProperty propagatedLsp =
getPropagatedProperty(lsp, gby);
- if (propagatedLsp != null) {
- propsLocal.add(propagatedLsp);
+ ILogicalOperator childOp = gby.getInputs().get(0).getValue();
+ IPhysicalPropertiesVector childProperties =
childOp.getDeliveredPhysicalProperties();
+ IPartitioningProperty partitioning =
+ computePartitioningProperty(gby,
childProperties.getPartitioningProperty());
+ List<ILocalStructuralProperty> local = computeLocalProperties(gby,
childProperties.getLocalProperties());
+ deliveredProperties = new StructuralPropertiesVector(partitioning,
local);
+ }
+
+ private IPartitioningProperty computePartitioningProperty(GroupByOperator
gby,
+ IPartitioningProperty childPartitioning) {
+ Map<LogicalVariable, LogicalVariable> substMap =
+ computePartitioningPropertySubstitutionMap(gby,
childPartitioning);
+ return substMap != null ?
childPartitioning.substituteColumnVars(substMap) : childPartitioning;
+ }
+
+ private List<ILocalStructuralProperty>
computeLocalProperties(GroupByOperator gby,
+ List<ILocalStructuralProperty> childLocals) {
+ List<ILocalStructuralProperty> propsLocal = new ArrayList<>();
+ if (childLocals != null) {
+ for (ILocalStructuralProperty lsp : childLocals) {
+ ILocalStructuralProperty propagatedLsp =
getPropagatedProperty(lsp, gby);
+ if (propagatedLsp != null) {
+ propsLocal.add(propagatedLsp);
+ }
}
}
- deliveredProperties = new StructuralPropertiesVector(pp, propsLocal);
+ return propsLocal;
}
// If we have "gby var1 as var3, var2 as var4"
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IntersectPOperator.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IntersectPOperator.java
index 3cf0fb9..9a595c5 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IntersectPOperator.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IntersectPOperator.java
@@ -89,7 +89,7 @@ public class IntersectPOperator extends
AbstractPhysicalOperator {
@Override
public void computeDeliveredProperties(ILogicalOperator iop,
IOptimizationContext context) {
IntersectOperator op = (IntersectOperator) iop;
- IPartitioningProperty pp =
+ IPartitioningProperty childpp =
op.getInputs().get(0).getValue().getDeliveredPhysicalProperties().getPartitioningProperty();
List<LogicalVariable> outputCompareVars =
op.getOutputCompareVariables();
@@ -98,7 +98,7 @@ public class IntersectPOperator extends
AbstractPhysicalOperator {
for (int i = 0; i < numCompareVars; i++) {
varMaps.put(op.getInputCompareVariables(0).get(i),
outputCompareVars.get(i));
}
- pp.substituteColumnVars(varMaps);
+ IPartitioningProperty pp = childpp.substituteColumnVars(varMaps);
List<ILocalStructuralProperty> propsLocal = new ArrayList<>();
List<OrderColumn> orderColumns = new ArrayList<>();
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java
index 3e78fd2..3a7f742 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java
@@ -59,8 +59,8 @@ public class BroadcastPartitioningProperty implements
IPartitioningProperty {
}
@Override
- public void substituteColumnVars(Map<LogicalVariable, LogicalVariable>
varMap) {
-
+ public IPartitioningProperty substituteColumnVars(Map<LogicalVariable,
LogicalVariable> varMap) {
+ return this;
}
@Override
@@ -68,4 +68,8 @@ public class BroadcastPartitioningProperty implements
IPartitioningProperty {
return new BroadcastPartitioningProperty(domain);
}
+ @Override
+ public String toString() {
+ return getPartitioningType().toString() + " domain:" + domain;
+ }
}
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java
index 5164192..f5eef1c 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java
@@ -76,7 +76,7 @@ public interface IPartitioningProperty extends
IStructuralProperty {
void setNodeDomain(INodeDomain domain);
- void substituteColumnVars(Map<LogicalVariable, LogicalVariable> varMap);
+ IPartitioningProperty substituteColumnVars(Map<LogicalVariable,
LogicalVariable> varMap);
IPartitioningProperty clonePartitioningProperty();
}
@@ -115,12 +115,13 @@ class UnpartitionedProperty implements
IPartitioningProperty {
}
@Override
- public void substituteColumnVars(Map<LogicalVariable, LogicalVariable>
variableMap) {
+ public IPartitioningProperty substituteColumnVars(Map<LogicalVariable,
LogicalVariable> variableMap) {
// No partition columns are maintained for UNPARTITIONED.
+ return UNPARTITIONED;
}
@Override
public IPartitioningProperty clonePartitioningProperty() {
- return new UnpartitionedProperty();
+ return UNPARTITIONED;
}
}
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java
index b5a2bb5..7112ef7 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java
@@ -55,7 +55,7 @@ public class OrderedPartitionedProperty implements
IPartitioningProperty {
@Override
public String toString() {
- return getPartitioningType().toString() + orderColumns;
+ return getPartitioningType().toString() + orderColumns + " domain:" +
domain;
}
@Override
@@ -84,17 +84,24 @@ public class OrderedPartitionedProperty implements
IPartitioningProperty {
}
@Override
- public void substituteColumnVars(Map<LogicalVariable, LogicalVariable>
varMap) {
+ public IPartitioningProperty substituteColumnVars(Map<LogicalVariable,
LogicalVariable> varMap) {
+ boolean applied = false;
+ List<OrderColumn> newOrderColumns = new
ArrayList<>(orderColumns.size());
for (OrderColumn orderColumn : orderColumns) {
- if (varMap.containsKey(orderColumn.getColumn())) {
- orderColumn.setColumn(varMap.get(orderColumn.getColumn()));
+ LogicalVariable columnVar = orderColumn.getColumn();
+ LogicalVariable newColumnVar = varMap.get(columnVar);
+ if (newColumnVar != null) {
+ applied = true;
+ } else {
+ newColumnVar = columnVar;
}
+ newOrderColumns.add(new OrderColumn(newColumnVar,
orderColumn.getOrder()));
}
+ return applied ? new OrderedPartitionedProperty(newOrderColumns,
domain) : this;
}
@Override
public IPartitioningProperty clonePartitioningProperty() {
return new OrderedPartitionedProperty(new ArrayList<>(orderColumns),
domain);
}
-
}
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java
index 951a031..f5c7aa8 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java
@@ -64,8 +64,8 @@ public class RandomPartitioningProperty implements
IPartitioningProperty {
}
@Override
- public void substituteColumnVars(Map<LogicalVariable, LogicalVariable>
varMap) {
-
+ public IPartitioningProperty substituteColumnVars(Map<LogicalVariable,
LogicalVariable> varMap) {
+ return this;
}
@Override
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java
index 5966407..fa8650c 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java
@@ -51,7 +51,7 @@ public final class UnorderedPartitionedProperty extends
AbstractGroupingProperty
@Override
public String toString() {
- return getPartitioningType().toString() + columnSet;
+ return getPartitioningType().toString() + columnSet + " domain:" +
domain;
}
@Override
@@ -70,12 +70,16 @@ public final class UnorderedPartitionedProperty extends
AbstractGroupingProperty
}
@Override
- public void substituteColumnVars(Map<LogicalVariable, LogicalVariable>
varMap) {
- varMap.forEach((key, value) -> {
- if (columnSet.remove(key)) {
- columnSet.add(value);
+ public IPartitioningProperty substituteColumnVars(Map<LogicalVariable,
LogicalVariable> varMap) {
+ boolean applied = false;
+ Set<LogicalVariable> newColumnSet = new ListSet<>(columnSet);
+ for (Map.Entry<LogicalVariable, LogicalVariable> me :
varMap.entrySet()) {
+ if (newColumnSet.remove(me.getKey())) {
+ newColumnSet.add(me.getValue());
+ applied = true;
}
- });
+ }
+ return applied ? new UnorderedPartitionedProperty(newColumnSet,
domain) : this;
}
@Override
diff --git
a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
index 6ba6a0f..afbbc4f 100644
---
a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
+++
b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
@@ -84,6 +84,7 @@ public class PullSelectOutOfEqJoin implements
IAlgebraicRewriteRule {
ILogicalExpression newJoinCond = makeCondition(eqVarVarComps, context);
join.getCondition().setValue(newJoinCond);
select.getInputs().add(new MutableObject<ILogicalOperator>(join));
+ select.recomputeSchema();
opRef.setValue(select);
context.computeAndSetTypeEnvironmentForOperator(select);
return true;
