Luo Chen has uploaded a new change for review. https://asterix-gerrit.ics.uci.edu/2964
Change subject: [ASTERIXDB-2174] Fix partitioning improvement of group by ...................................................................... [ASTERIXDB-2174] Fix partitioning improvement of group by - user model changes: no - storage format changes: no - interface changes: no Details: - Fix the improvement of partitioning propagation of group by introduced by https://asterix-gerrit.ics.uci.edu/#/c/2176/. - The problem is that we cannot modify the partitioning property of the input operator directly, since that property is owned by the input. Otherwise, the property of the child operator would be modified even before the group by operator, and this caused some tests regarding common sub operator extration to fail (introducing REPLICATE). Change-Id: I874efd283d637a424119e32e467aaf77fd102194 --- M asterixdb/asterix-app/src/test/resources/optimizerts/results/split-materialization-above-join.plan M asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan M asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581.plan M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IntersectPOperator.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java 10 files changed, 201 insertions(+), 183 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/64/2964/1 diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/split-materialization-above-join.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/split-materialization-above-join.plan index 22bc323..a81a142 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/split-materialization-above-join.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/split-materialization-above-join.plan @@ -32,26 +32,26 @@ -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- HYBRID_HASH_JOIN [$$prefixTokenLeft][$$prefixTokenRight] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$prefixTokenLeft] |PARTITIONED| - -- UNNEST |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- PRE_CLUSTERED_GROUP_BY[$$95] |PARTITIONED| - { - -- AGGREGATE |LOCAL| - -- STREAM_SELECT |LOCAL| - -- NESTED_TUPLE_SOURCE |LOCAL| - } - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STABLE_SORT [$$95(ASC), $$i(ASC)] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$95] |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- HYBRID_HASH_JOIN [$$tokenUnranked][$$tokenGroupped] |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$prefixTokenRight] |PARTITIONED| + -- UNNEST |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- PRE_CLUSTERED_GROUP_BY[$$97] |PARTITIONED| + { + -- AGGREGATE |LOCAL| + -- STREAM_SELECT |LOCAL| + -- NESTED_TUPLE_SOURCE |LOCAL| + } + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$97(ASC), $$i(ASC)] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$97] |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| + -- HYBRID_HASH_JOIN [$$tokenUnranked][$$tokenGroupped] |PARTITIONED| -- HASH_PARTITION_EXCHANGE [$$tokenUnranked] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- UNNEST |PARTITIONED| @@ -62,24 +62,19 @@ -- DATASOURCE_SCAN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- EMPTY_TUPLE_SOURCE |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$tokenGroupped] |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- RUNNING_AGGREGATE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- SORT_MERGE_EXCHANGE [$$99(ASC), $$tokenGroupped(ASC) ] |PARTITIONED| - -- STABLE_SORT [$$99(ASC), $$tokenGroupped(ASC)] |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- SORT_GROUP_BY[$$128] |PARTITIONED| - { - -- AGGREGATE |LOCAL| - -- NESTED_TUPLE_SOURCE |LOCAL| - } - -- HASH_PARTITION_EXCHANGE [$$128] |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ASSIGN |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$tokenGroupped] |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- RUNNING_AGGREGATE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- SORT_MERGE_EXCHANGE [$$102(ASC), $$tokenGroupped(ASC) ] |PARTITIONED| + -- STABLE_SORT [$$102(ASC), $$tokenGroupped(ASC)] |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- SORT_GROUP_BY[$$130] |PARTITIONED| + { + -- AGGREGATE |LOCAL| + -- NESTED_TUPLE_SOURCE |LOCAL| + } + -- HASH_PARTITION_EXCHANGE [$$130] |PARTITIONED| -- SORT_GROUP_BY[$$token] |PARTITIONED| { -- AGGREGATE |LOCAL| @@ -97,23 +92,23 @@ -- DATASOURCE_SCAN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- EMPTY_TUPLE_SOURCE |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$prefixTokenRight] |PARTITIONED| - -- UNNEST |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- PRE_CLUSTERED_GROUP_BY[$$97] |PARTITIONED| - { - -- AGGREGATE |LOCAL| - -- STREAM_SELECT |LOCAL| - -- NESTED_TUPLE_SOURCE |LOCAL| - } + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$prefixTokenRight] |PARTITIONED| + -- UNNEST |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STABLE_SORT [$$97(ASC), $$i(ASC)] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$97] |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- HYBRID_HASH_JOIN [$$tokenUnranked][$$tokenGroupped] |PARTITIONED| + -- PRE_CLUSTERED_GROUP_BY[$$97] |PARTITIONED| + { + -- AGGREGATE |LOCAL| + -- STREAM_SELECT |LOCAL| + -- NESTED_TUPLE_SOURCE |LOCAL| + } + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STABLE_SORT [$$97(ASC), $$i(ASC)] |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$97] |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| + -- HYBRID_HASH_JOIN [$$tokenUnranked][$$tokenGroupped] |PARTITIONED| -- HASH_PARTITION_EXCHANGE [$$tokenUnranked] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- UNNEST |PARTITIONED| @@ -124,21 +119,19 @@ -- DATASOURCE_SCAN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- EMPTY_TUPLE_SOURCE |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$tokenGroupped] |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- RUNNING_AGGREGATE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- SORT_MERGE_EXCHANGE [$$102(ASC), $$tokenGroupped(ASC) ] |PARTITIONED| - -- STABLE_SORT [$$102(ASC), $$tokenGroupped(ASC)] |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- SORT_GROUP_BY[$$130] |PARTITIONED| - { - -- AGGREGATE |LOCAL| - -- NESTED_TUPLE_SOURCE |LOCAL| - } - -- HASH_PARTITION_EXCHANGE [$$130] |PARTITIONED| - -- REPLICATE |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$tokenGroupped] |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- RUNNING_AGGREGATE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- SORT_MERGE_EXCHANGE [$$102(ASC), $$tokenGroupped(ASC) ] |PARTITIONED| + -- STABLE_SORT [$$102(ASC), $$tokenGroupped(ASC)] |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- SORT_GROUP_BY[$$130] |PARTITIONED| + { + -- AGGREGATE |LOCAL| + -- NESTED_TUPLE_SOURCE |LOCAL| + } + -- HASH_PARTITION_EXCHANGE [$$130] |PARTITIONED| -- SORT_GROUP_BY[$$token] |PARTITIONED| { -- AGGREGATE |LOCAL| diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan index c636836..fff35d0 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan @@ -51,16 +51,19 @@ -- HYBRID_HASH_JOIN [$$120][$$130] |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ASSIGN |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- BTREE_SEARCH |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- BTREE_SEARCH |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| @@ -83,50 +86,49 @@ -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- HYBRID_HASH_JOIN [$$134][$$133] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$134] |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- STREAM_SELECT |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- PRE_CLUSTERED_GROUP_BY[$$142] |PARTITIONED| - { - -- AGGREGATE |LOCAL| - -- AGGREGATE |LOCAL| - -- STREAM_SELECT |LOCAL| - -- NESTED_TUPLE_SOURCE |LOCAL| - } - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$171] |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- HYBRID_HASH_JOIN [$$142][$$144] |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ASSIGN |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- PRE_CLUSTERED_GROUP_BY[$$176] |PARTITIONED| + { + -- AGGREGATE |LOCAL| + -- AGGREGATE |LOCAL| + -- STREAM_SELECT |LOCAL| + -- NESTED_TUPLE_SOURCE |LOCAL| + } + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| + -- HYBRID_HASH_JOIN [$$176][$$177] |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- BTREE_SEARCH |PARTITIONED| + -- REPLICATE |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- EMPTY_TUPLE_SOURCE |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$177] |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ASSIGN |PARTITIONED| + -- BTREE_SEARCH |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$177] |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| + -- REPLICATE |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- DATASOURCE_SCAN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- DATASOURCE_SCAN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| -- HASH_PARTITION_EXCHANGE [$$133] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| @@ -178,7 +180,7 @@ -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- HYBRID_HASH_JOIN [$$158][$$159] |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| + -- REPLICATE |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| @@ -211,44 +213,46 @@ -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- HYBRID_HASH_JOIN [$$171][$$170] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$171] |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$171] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| - -- STREAM_SELECT |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- PRE_CLUSTERED_GROUP_BY[$$176] |PARTITIONED| - { - -- AGGREGATE |LOCAL| - -- AGGREGATE |LOCAL| - -- STREAM_SELECT |LOCAL| - -- NESTED_TUPLE_SOURCE |LOCAL| - } + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| + -- PRE_CLUSTERED_GROUP_BY[$$176] |PARTITIONED| + { + -- AGGREGATE |LOCAL| + -- AGGREGATE |LOCAL| + -- STREAM_SELECT |LOCAL| + -- NESTED_TUPLE_SOURCE |LOCAL| + } -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- HYBRID_HASH_JOIN [$$176][$$177] |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| + -- HYBRID_HASH_JOIN [$$176][$$177] |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- BTREE_SEARCH |PARTITIONED| + -- REPLICATE |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- EMPTY_TUPLE_SOURCE |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$177] |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ASSIGN |PARTITIONED| + -- BTREE_SEARCH |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- REPLICATE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$177] |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| + -- REPLICATE |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- DATASOURCE_SCAN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- DATASOURCE_SCAN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| -- HASH_PARTITION_EXCHANGE [$$170] |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- ASSIGN |PARTITIONED| diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581.plan index 38d0b5f..d0afd76 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581.plan +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581.plan @@ -45,24 +45,29 @@ -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- HYBRID_HASH_JOIN [$$85][$$142] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$85] |PARTITIONED| - -- STREAM_PROJECT |UNPARTITIONED| - -- ASSIGN |UNPARTITIONED| - -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| - -- REPLICATE |UNPARTITIONED| - -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| - -- AGGREGATE |UNPARTITIONED| - -- AGGREGATE |UNPARTITIONED| - -- RANDOM_MERGE_EXCHANGE |PARTITIONED| - -- AGGREGATE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- STREAM_SELECT |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- DATASOURCE_SCAN |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$151] |PARTITIONED| + -- STREAM_PROJECT |UNPARTITIONED| + -- ASSIGN |UNPARTITIONED| + -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| + -- REPLICATE |UNPARTITIONED| + -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| + -- AGGREGATE |UNPARTITIONED| + -- AGGREGATE |UNPARTITIONED| + -- RANDOM_MERGE_EXCHANGE |PARTITIONED| + -- AGGREGATE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| -- HASH_PARTITION_EXCHANGE [$$142] |PARTITIONED| -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| @@ -122,24 +127,26 @@ -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- HYBRID_HASH_JOIN [$$151][$$152] |PARTITIONED| - -- HASH_PARTITION_EXCHANGE [$$151] |PARTITIONED| - -- STREAM_PROJECT |UNPARTITIONED| - -- ASSIGN |UNPARTITIONED| - -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| - -- REPLICATE |UNPARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- REPLICATE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$151] |PARTITIONED| + -- STREAM_PROJECT |UNPARTITIONED| + -- ASSIGN |UNPARTITIONED| -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| - -- AGGREGATE |UNPARTITIONED| - -- AGGREGATE |UNPARTITIONED| - -- RANDOM_MERGE_EXCHANGE |PARTITIONED| - -- AGGREGATE |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- STREAM_SELECT |PARTITIONED| - -- ASSIGN |PARTITIONED| - -- STREAM_PROJECT |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- DATASOURCE_SCAN |PARTITIONED| + -- REPLICATE |UNPARTITIONED| + -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| + -- AGGREGATE |UNPARTITIONED| + -- AGGREGATE |UNPARTITIONED| + -- RANDOM_MERGE_EXCHANGE |PARTITIONED| + -- AGGREGATE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- STREAM_SELECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| - -- EMPTY_TUPLE_SOURCE |PARTITIONED| + -- DATASOURCE_SCAN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| -- HASH_PARTITION_EXCHANGE [$$152] |PARTITIONED| -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java index 64e50ed..a6d7201 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java @@ -89,7 +89,10 @@ IPartitioningProperty pp = childProp.getPartitioningProperty(); Map<LogicalVariable, LogicalVariable> ppSubstMap = computePartitioningPropertySubstitutionMap(gby, pp); if (ppSubstMap != null) { - pp.substituteColumnVars(ppSubstMap); + // We cannot modify pp directly, since it is owned by the input operator. + // Otherwise, the partitioning property would be modified even before this group by operator, + // which will be undesirable. + pp = pp.substituteColumnVars(ppSubstMap, false); } List<ILocalStructuralProperty> childLocals = childProp.getLocalProperties(); if (childLocals == null) { diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IntersectPOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IntersectPOperator.java index 544c546..1d231ba 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IntersectPOperator.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IntersectPOperator.java @@ -94,7 +94,7 @@ for (int i = 0; i < op.getOutputVars().size(); i++) { varMaps.put(op.getInputVariables(0).get(i), op.getOutputVars().get(i)); } - pp.substituteColumnVars(varMaps); + pp = pp.substituteColumnVars(varMaps, false); List<ILocalStructuralProperty> propsLocal = new ArrayList<>(); List<OrderColumn> orderColumns = new ArrayList<>(); diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java index bc6a45d..a90bfe2 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java @@ -59,7 +59,8 @@ } @Override - public void substituteColumnVars(Map<LogicalVariable, LogicalVariable> varMap) { + public IPartitioningProperty substituteColumnVars(Map<LogicalVariable, LogicalVariable> varMap, boolean inplace) { + return this; } } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java index f41d197..fc1cd09 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java @@ -76,7 +76,7 @@ void setNodeDomain(INodeDomain domain); - void substituteColumnVars(Map<LogicalVariable, LogicalVariable> varMap); + IPartitioningProperty substituteColumnVars(Map<LogicalVariable, LogicalVariable> varMap, boolean inplace); } class UnpartitionedProperty implements IPartitioningProperty { @@ -113,7 +113,9 @@ } @Override - public void substituteColumnVars(Map<LogicalVariable, LogicalVariable> variableMap) { + public IPartitioningProperty substituteColumnVars(Map<LogicalVariable, LogicalVariable> variableMap, + boolean inplace) { // No partition columns are maintained for UNPARTITIONED. + return this; } } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java index 23c8273..23493ab2 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java @@ -84,12 +84,15 @@ } @Override - public void substituteColumnVars(Map<LogicalVariable, LogicalVariable> varMap) { - for (OrderColumn orderColumn : orderColumns) { + public IPartitioningProperty substituteColumnVars(Map<LogicalVariable, LogicalVariable> varMap, boolean inplace) { + OrderedPartitionedProperty resultProperty = + inplace ? this : new OrderedPartitionedProperty(new ArrayList<>(orderColumns), domain); + for (OrderColumn orderColumn : resultProperty.orderColumns) { if (varMap.containsKey(orderColumn.getColumn())) { orderColumn.setColumn(varMap.get(orderColumn.getColumn())); } } + return resultProperty; } } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java index bbd835c..762a6e5 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java @@ -64,7 +64,8 @@ } @Override - public void substituteColumnVars(Map<LogicalVariable, LogicalVariable> varMap) { + public IPartitioningProperty substituteColumnVars(Map<LogicalVariable, LogicalVariable> varMap, boolean inplace) { + return this; } } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java index f59638c..78384d4 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java @@ -19,6 +19,7 @@ package org.apache.hyracks.algebricks.core.algebra.properties; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -69,12 +70,15 @@ } @Override - public void substituteColumnVars(Map<LogicalVariable, LogicalVariable> varMap) { + public IPartitioningProperty substituteColumnVars(Map<LogicalVariable, LogicalVariable> varMap, boolean inplace) { + UnorderedPartitionedProperty resultProperty = inplace? + this:new UnorderedPartitionedProperty(new HashSet<>(columnSet), domain); varMap.forEach((key, value) -> { - if (columnSet.remove(key)) { - columnSet.add(value); + if (resultProperty.columnSet.remove(key)) { + resultProperty.columnSet.add(value); } }); + return resultProperty; } } -- To view, visit https://asterix-gerrit.ics.uci.edu/2964 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I874efd283d637a424119e32e467aaf77fd102194 Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen <[email protected]>
