Repository: asterixdb Updated Branches: refs/heads/master b71d9ff7b -> 62ab70d37
[ASTERIXDB-2174] Fix partitioning improvement of group by - user model changes: no - storage format changes: no - interface changes: no Details: - Fix the improvement of partitioning propagation of group by introduced by https://asterix-gerrit.ics.uci.edu/#/c/2176/. - The problem is that we cannot modify the partitioning property of the input operator directly, since that property is owned by the input. Otherwise, the property of the child operator would be modified even before the group by operator, and this caused some tests regarding common sub operator extration to fail (introducing REPLICATE). Change-Id: I81a74ab1242772c790175414281e3f200537b386 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2966 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Dmitry Lychagin <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/5e24a1bf Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/5e24a1bf Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/5e24a1bf Branch: refs/heads/master Commit: 5e24a1bfd115378eb0d705519d30b0028127e8c0 Parents: 142aad0 Author: luochen01 <[email protected]> Authored: Thu Sep 13 14:14:29 2018 -0700 Committer: Dmitry Lychagin <[email protected]> Committed: Thu Sep 13 16:15:01 2018 -0700 ---------------------------------------------------------------------- .../split-materialization-above-join.plan | 125 ++++++++-------- .../tpcds/query-ASTERIXDB-1581-correlated.plan | 148 ++++++++++--------- .../results/tpcds/query-ASTERIXDB-1581.plan | 75 +++++----- .../AbstractPreclusteredGroupByPOperator.java | 4 + .../BroadcastPartitioningProperty.java | 6 + .../properties/IPartitioningProperty.java | 7 + .../properties/OrderedPartitionedProperty.java | 7 +- .../properties/RandomPartitioningProperty.java | 6 + .../UnorderedPartitionedProperty.java | 6 + 9 files changed, 211 insertions(+), 173 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e24a1bf/asterixdb/asterix-app/src/test/resources/optimizerts/results/split-materialization-above-join.plan ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/split-materialization-above-join.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/split-materialization-above-join.plan index 22bc323..a81a142 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/split-materialization-above-join.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/split-materialization-above-join.plan @@ -32,26 +32,26 @@ -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- HYBRID_HASH_JOIN [$$prefixTokenLeft][$$prefixTokenRight] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$prefixTokenLeft] |PARTITIONED| - -- UNNEST |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- PRE_CLUSTERED_GROUP_BY[$$95] |PARTITIONED| - { - -- AGGREGATE |LOCAL| - -- STREAM_SELECT |LOCAL| - -- NESTED_TUPLE_SOURCE |LOCAL| - } - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STABLE_SORT [$$95(ASC), $$i(ASC)] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$95] |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- HYBRID_HASH_JOIN [$$tokenUnranked][$$tokenGroupped] |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$prefixTokenRight] |PARTITIONED| + -- UNNEST |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- PRE_CLUSTERED_GROUP_BY[$$97] |PARTITIONED| + { + -- AGGREGATE |LOCAL| + -- STREAM_SELECT |LOCAL| + -- NESTED_TUPLE_SOURCE |LOCAL| + } + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$97(ASC), $$i(ASC)] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$97] |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| + -- HYBRID_HASH_JOIN [$$tokenUnranked][$$tokenGroupped] |PARTITIONED| -- HASH_PARTITION_EXCHANGE [$$tokenUnranked] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- UNNEST |PARTITIONED| @@ -62,24 +62,19 @@ -- DATASOURCE_SCAN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- EMPTY_TUPLE_SOURCE |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$tokenGroupped] |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- RUNNING_AGGREGATE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- SORT_MERGE_EXCHANGE [$$99(ASC), $$tokenGroupped(ASC) ] |PARTITIONED| - -- STABLE_SORT [$$99(ASC), $$tokenGroupped(ASC)] |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- SORT_GROUP_BY[$$128] |PARTITIONED| - { - -- AGGREGATE |LOCAL| - -- NESTED_TUPLE_SOURCE |LOCAL| - } - -- HASH_PARTITION_EXCHANGE [$$128] |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ASSIGN |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$tokenGroupped] |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- RUNNING_AGGREGATE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- SORT_MERGE_EXCHANGE [$$102(ASC), $$tokenGroupped(ASC) ] |PARTITIONED| + -- STABLE_SORT [$$102(ASC), $$tokenGroupped(ASC)] |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- SORT_GROUP_BY[$$130] |PARTITIONED| + { + -- AGGREGATE |LOCAL| + -- NESTED_TUPLE_SOURCE |LOCAL| + } + -- HASH_PARTITION_EXCHANGE [$$130] |PARTITIONED| -- SORT_GROUP_BY[$$token] |PARTITIONED| { -- AGGREGATE |LOCAL| @@ -97,23 +92,23 @@ -- DATASOURCE_SCAN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- EMPTY_TUPLE_SOURCE |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$prefixTokenRight] |PARTITIONED| - -- UNNEST |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- PRE_CLUSTERED_GROUP_BY[$$97] |PARTITIONED| - { - -- AGGREGATE |LOCAL| - -- STREAM_SELECT |LOCAL| - -- NESTED_TUPLE_SOURCE |LOCAL| - } + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$prefixTokenRight] |PARTITIONED| + -- UNNEST |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STABLE_SORT [$$97(ASC), $$i(ASC)] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$97] |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- HYBRID_HASH_JOIN [$$tokenUnranked][$$tokenGroupped] |PARTITIONED| + -- PRE_CLUSTERED_GROUP_BY[$$97] |PARTITIONED| + { + -- AGGREGATE |LOCAL| + -- STREAM_SELECT |LOCAL| + -- NESTED_TUPLE_SOURCE |LOCAL| + } + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$97(ASC), $$i(ASC)] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$97] |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| + -- HYBRID_HASH_JOIN [$$tokenUnranked][$$tokenGroupped] |PARTITIONED| -- HASH_PARTITION_EXCHANGE [$$tokenUnranked] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- UNNEST |PARTITIONED| @@ -124,21 +119,19 @@ -- DATASOURCE_SCAN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- EMPTY_TUPLE_SOURCE |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$tokenGroupped] |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- RUNNING_AGGREGATE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- SORT_MERGE_EXCHANGE [$$102(ASC), $$tokenGroupped(ASC) ] |PARTITIONED| - -- STABLE_SORT [$$102(ASC), $$tokenGroupped(ASC)] |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- SORT_GROUP_BY[$$130] |PARTITIONED| - { - -- AGGREGATE |LOCAL| - -- NESTED_TUPLE_SOURCE |LOCAL| - } - -- HASH_PARTITION_EXCHANGE [$$130] |PARTITIONED| - -- REPLICATE |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$tokenGroupped] |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- RUNNING_AGGREGATE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- SORT_MERGE_EXCHANGE [$$102(ASC), $$tokenGroupped(ASC) ] |PARTITIONED| + -- STABLE_SORT [$$102(ASC), $$tokenGroupped(ASC)] |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- SORT_GROUP_BY[$$130] |PARTITIONED| + { + -- AGGREGATE |LOCAL| + -- NESTED_TUPLE_SOURCE |LOCAL| + } + -- HASH_PARTITION_EXCHANGE [$$130] |PARTITIONED| -- SORT_GROUP_BY[$$token] |PARTITIONED| { -- AGGREGATE |LOCAL| http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e24a1bf/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan index c636836..fff35d0 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan @@ -51,16 +51,19 @@ -- HYBRID_HASH_JOIN [$$120][$$130] |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ASSIGN |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- BTREE_SEARCH |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BTREE_SEARCH |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| @@ -83,50 +86,49 @@ -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- HYBRID_HASH_JOIN [$$134][$$133] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$134] |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- STREAM_SELECT |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- PRE_CLUSTERED_GROUP_BY[$$142] |PARTITIONED| - { - -- AGGREGATE |LOCAL| - -- AGGREGATE |LOCAL| - -- STREAM_SELECT |LOCAL| - -- NESTED_TUPLE_SOURCE |LOCAL| - } - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$171] |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- HYBRID_HASH_JOIN [$$142][$$144] |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ASSIGN |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- PRE_CLUSTERED_GROUP_BY[$$176] |PARTITIONED| + { + -- AGGREGATE |LOCAL| + -- AGGREGATE |LOCAL| + -- STREAM_SELECT |LOCAL| + -- NESTED_TUPLE_SOURCE |LOCAL| + } + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| + -- HYBRID_HASH_JOIN [$$176][$$177] |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- BTREE_SEARCH |PARTITIONED| + -- REPLICATE |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- EMPTY_TUPLE_SOURCE |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$177] |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ASSIGN |PARTITIONED| + -- BTREE_SEARCH |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$177] |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| + -- REPLICATE |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- DATASOURCE_SCAN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- DATASOURCE_SCAN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| -- HASH_PARTITION_EXCHANGE [$$133] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| @@ -178,7 +180,7 @@ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- HYBRID_HASH_JOIN [$$158][$$159] |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| + -- REPLICATE |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| @@ -211,44 +213,46 @@ -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- HYBRID_HASH_JOIN [$$171][$$170] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$171] |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$171] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| - -- STREAM_SELECT |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- PRE_CLUSTERED_GROUP_BY[$$176] |PARTITIONED| - { - -- AGGREGATE |LOCAL| - -- AGGREGATE |LOCAL| - -- STREAM_SELECT |LOCAL| - -- NESTED_TUPLE_SOURCE |LOCAL| - } + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| + -- PRE_CLUSTERED_GROUP_BY[$$176] |PARTITIONED| + { + -- AGGREGATE |LOCAL| + -- AGGREGATE |LOCAL| + -- STREAM_SELECT |LOCAL| + -- NESTED_TUPLE_SOURCE |LOCAL| + } -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- HYBRID_HASH_JOIN [$$176][$$177] |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| + -- HYBRID_HASH_JOIN [$$176][$$177] |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- BTREE_SEARCH |PARTITIONED| + -- REPLICATE |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- EMPTY_TUPLE_SOURCE |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$177] |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ASSIGN |PARTITIONED| + -- BTREE_SEARCH |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$177] |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| + -- REPLICATE |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- DATASOURCE_SCAN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- DATASOURCE_SCAN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| -- HASH_PARTITION_EXCHANGE [$$170] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e24a1bf/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581.plan ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581.plan index 38d0b5f..d0afd76 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581.plan @@ -45,24 +45,29 @@ -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- HYBRID_HASH_JOIN [$$85][$$142] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$85] |PARTITIONED| - -- STREAM_PROJECT |UNPARTITIONED| - -- ASSIGN |UNPARTITIONED| - -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| - -- REPLICATE |UNPARTITIONED| - -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| - -- AGGREGATE |UNPARTITIONED| - -- AGGREGATE |UNPARTITIONED| - -- RANDOM_MERGE_EXCHANGE |PARTITIONED| - -- AGGREGATE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- STREAM_SELECT |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- DATASOURCE_SCAN |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$151] |PARTITIONED| + -- STREAM_PROJECT |UNPARTITIONED| + -- ASSIGN |UNPARTITIONED| + -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| + -- REPLICATE |UNPARTITIONED| + -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| + -- AGGREGATE |UNPARTITIONED| + -- AGGREGATE |UNPARTITIONED| + -- RANDOM_MERGE_EXCHANGE |PARTITIONED| + -- AGGREGATE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| -- HASH_PARTITION_EXCHANGE [$$142] |PARTITIONED| -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| @@ -122,24 +127,26 @@ -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- HYBRID_HASH_JOIN [$$151][$$152] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$151] |PARTITIONED| - -- STREAM_PROJECT |UNPARTITIONED| - -- ASSIGN |UNPARTITIONED| - -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| - -- REPLICATE |UNPARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$151] |PARTITIONED| + -- STREAM_PROJECT |UNPARTITIONED| + -- ASSIGN |UNPARTITIONED| -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| - -- AGGREGATE |UNPARTITIONED| - -- AGGREGATE |UNPARTITIONED| - -- RANDOM_MERGE_EXCHANGE |PARTITIONED| - -- AGGREGATE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- STREAM_SELECT |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- DATASOURCE_SCAN |PARTITIONED| + -- REPLICATE |UNPARTITIONED| + -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| + -- AGGREGATE |UNPARTITIONED| + -- AGGREGATE |UNPARTITIONED| + -- RANDOM_MERGE_EXCHANGE |PARTITIONED| + -- AGGREGATE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- DATASOURCE_SCAN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| -- HASH_PARTITION_EXCHANGE [$$152] |PARTITIONED| -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e24a1bf/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java index 64e50ed..c18d76c 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java @@ -89,6 +89,10 @@ public abstract class AbstractPreclusteredGroupByPOperator extends AbstractPhysi IPartitioningProperty pp = childProp.getPartitioningProperty(); Map<LogicalVariable, LogicalVariable> ppSubstMap = computePartitioningPropertySubstitutionMap(gby, pp); if (ppSubstMap != null) { + // We cannot modify pp directly, since it is owned by the input operator. + // Otherwise, the partitioning property would be modified even before this group by operator, + // which will be undesirable. + pp = pp.clonePartitioningProperty(); pp.substituteColumnVars(ppSubstMap); } List<ILocalStructuralProperty> childLocals = childProp.getLocalProperties(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e24a1bf/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java index bc6a45d..3e78fd2 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java @@ -60,6 +60,12 @@ public class BroadcastPartitioningProperty implements IPartitioningProperty { @Override public void substituteColumnVars(Map<LogicalVariable, LogicalVariable> varMap) { + + } + + @Override + public IPartitioningProperty clonePartitioningProperty() { + return new BroadcastPartitioningProperty(domain); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e24a1bf/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java index f41d197..5164192 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java @@ -77,6 +77,8 @@ public interface IPartitioningProperty extends IStructuralProperty { void setNodeDomain(INodeDomain domain); void substituteColumnVars(Map<LogicalVariable, LogicalVariable> varMap); + + IPartitioningProperty clonePartitioningProperty(); } class UnpartitionedProperty implements IPartitioningProperty { @@ -116,4 +118,9 @@ class UnpartitionedProperty implements IPartitioningProperty { public void substituteColumnVars(Map<LogicalVariable, LogicalVariable> variableMap) { // No partition columns are maintained for UNPARTITIONED. } + + @Override + public IPartitioningProperty clonePartitioningProperty() { + return new UnpartitionedProperty(); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e24a1bf/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java index 23c8273..b5a2bb5 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java @@ -28,7 +28,7 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; public class OrderedPartitionedProperty implements IPartitioningProperty { - private List<OrderColumn> orderColumns; + private final List<OrderColumn> orderColumns; private INodeDomain domain; public OrderedPartitionedProperty(List<OrderColumn> orderColumns, INodeDomain domain) { @@ -92,4 +92,9 @@ public class OrderedPartitionedProperty implements IPartitioningProperty { } } + @Override + public IPartitioningProperty clonePartitioningProperty() { + return new OrderedPartitionedProperty(new ArrayList<>(orderColumns), domain); + } + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e24a1bf/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java index bbd835c..951a031 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java @@ -65,6 +65,12 @@ public class RandomPartitioningProperty implements IPartitioningProperty { @Override public void substituteColumnVars(Map<LogicalVariable, LogicalVariable> varMap) { + + } + + @Override + public IPartitioningProperty clonePartitioningProperty() { + return new RandomPartitioningProperty(domain); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e24a1bf/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java index f59638c..e07cf15 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java @@ -19,6 +19,7 @@ package org.apache.hyracks.algebricks.core.algebra.properties; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -77,4 +78,9 @@ public final class UnorderedPartitionedProperty extends AbstractGroupingProperty }); } + @Override + public IPartitioningProperty clonePartitioningProperty() { + return new UnorderedPartitionedProperty(new HashSet<>(columnSet), domain); + } + }
