Repository: asterixdb Updated Branches: refs/heads/master 675244029 -> 42f17d2ba
[NO-ISSUE][COMP] Avoid adding redundant var in AbstractIntroduceGroupByCombinerRule - user model changes: no - storage format changes: no - interface changes: no For live variables added in new Group-by op, they should not be added again. Change-Id: Ic1ab9aee31db95d5782385bc3d53777da54f6d83 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2933 Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Dmitry Lychagin <dmitry.lycha...@couchbase.com> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/42f17d2b Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/42f17d2b Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/42f17d2b Branch: refs/heads/master Commit: 42f17d2bae4ef2db13af851f7d2bec03308c7dec Parents: 6752440 Author: Xikui Wang <xkk...@gmail.com> Authored: Wed Sep 5 10:51:07 2018 -0700 Committer: Xikui Wang <xkk...@gmail.com> Committed: Wed Sep 5 12:52:09 2018 -0700 ---------------------------------------------------------------------- .../redundant-var-in-groupby.1.ddl.sqlpp | 44 ++++++++++++++++++++ .../redundant-var-in-groupby.2.update.sqlpp | 25 +++++++++++ .../redundant-var-in-groupby.3.query.sqlpp | 20 +++++++++ .../redundant-var-in-groupby.4.ddl.sqlpp | 19 +++++++++ .../redundant-var-in-groupby.1.adm | 1 + .../resources/runtimets/testsuite_sqlpp.xml | 5 +++ .../AbstractIntroduceGroupByCombinerRule.java | 13 +++--- 7 files changed, 120 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/42f17d2b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/redundant-var-in-groupby/redundant-var-in-groupby.1.ddl.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/redundant-var-in-groupby/redundant-var-in-groupby.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/redundant-var-in-groupby/redundant-var-in-groupby.1.ddl.sqlpp new file mode 100644 index 0000000..3100af6 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/redundant-var-in-groupby/redundant-var-in-groupby.1.ddl.sqlpp @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +drop dataverse test if exists; +create dataverse test; +use test; + +create type TweetType as open { + id : int64 +}; + +create dataset Tweets(TweetType) primary key id; + +drop type NearbyBuildingType if exists; +create type NearbyBuildingType as open { + buildingId: int64, + buildingType : string, + buildingLocation : point +}; +create dataset NearbyBuildings(NearbyBuildingType) primary key buildingId; + +create function annotateTweet(x) { + LET nearby_buildings = (select r.buildingType as buildingType, count(r) as cnt + from NearbyBuildings r + where spatial_intersect(create_point(x.latitude, x.longitude), create_circle(r.buildingLocation, 3.0)) + group by r.buildingType) + select x.*, nearby_buildings +}; + http://git-wip-us.apache.org/repos/asf/asterixdb/blob/42f17d2b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/redundant-var-in-groupby/redundant-var-in-groupby.2.update.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/redundant-var-in-groupby/redundant-var-in-groupby.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/redundant-var-in-groupby/redundant-var-in-groupby.2.update.sqlpp new file mode 100644 index 0000000..ab13484 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/redundant-var-in-groupby/redundant-var-in-groupby.2.update.sqlpp @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; +insert into Tweets([{"id": 1, "text" : "im tweet", "latitude":0.0, "longitude" : 0.0}]); + +insert into NearbyBuildings([{"buildingId" : 0, "buildingType" : "school", "buildingLocation" : point("1, 1")}, +{"buildingId" : 1, "buildingType" : "school", "buildingLocation" : point("-1, 1")}, +{"buildingId" : 2, "buildingType" : "hospital", "buildingLocation" : point("1, -1")}]); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/42f17d2b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/redundant-var-in-groupby/redundant-var-in-groupby.3.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/redundant-var-in-groupby/redundant-var-in-groupby.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/redundant-var-in-groupby/redundant-var-in-groupby.3.query.sqlpp new file mode 100644 index 0000000..55a1c37 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/redundant-var-in-groupby/redundant-var-in-groupby.3.query.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + use test; +select value annotateTweet(t) from Tweets t; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/42f17d2b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/redundant-var-in-groupby/redundant-var-in-groupby.4.ddl.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/redundant-var-in-groupby/redundant-var-in-groupby.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/redundant-var-in-groupby/redundant-var-in-groupby.4.ddl.sqlpp new file mode 100644 index 0000000..d4b1d9d --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/redundant-var-in-groupby/redundant-var-in-groupby.4.ddl.sqlpp @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +drop dataverse test; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/42f17d2b/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/redundant-var-in-groupby/redundant-var-in-groupby.1.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/redundant-var-in-groupby/redundant-var-in-groupby.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/redundant-var-in-groupby/redundant-var-in-groupby.1.adm new file mode 100644 index 0000000..c563f9c --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/redundant-var-in-groupby/redundant-var-in-groupby.1.adm @@ -0,0 +1 @@ +[ { "nearby_buildings": [ { "buildingType": "hospital", "cnt": 1 }, { "buildingType": "school", "cnt": 2 } ], "id": 1, "text": "im tweet", "latitude": 0.0, "longitude": 0.0 } ] http://git-wip-us.apache.org/repos/asf/asterixdb/blob/42f17d2b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index b5fc053..6edc7bf 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -3220,6 +3220,11 @@ <expected-error>The byte size of a single group</expected-error> </compilation-unit> </test-case> + <test-case FilePath="group-by"> + <compilation-unit name="redundant-var-in-groupby"> + <output-dir compare="Text">redundant-var-in-groupby</output-dir> + </compilation-unit> + </test-case> </test-group> <test-group name="index-join"> <test-case FilePath="index-join"> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/42f17d2b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java index dc48d96..7b9c255 100644 --- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java +++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java @@ -84,10 +84,12 @@ public abstract class AbstractIntroduceGroupByCombinerRule extends AbstractIntro // p.second.getValue() should always return a VariableReferenceExpression, hence // usedDecorVars should always contain only one variable. p.second.getValue().getUsedVariables(usedDecorVars); - if (!newGbyLiveVars.contains(usedDecorVars.get(0))) { + LogicalVariable usedVar = usedDecorVars.get(0); + if (!newGbyLiveVars.contains(usedVar)) { // Let the left-hand side of gbyOp's decoration expressions populated through the combiner group-by without // any intermediate assignment. newGbyOp.addDecorExpression(null, p.second.getValue()); + newGbyLiveVars.add(usedVar); } } newGbyOp.setExecutionMode(ExecutionMode.LOCAL); @@ -97,14 +99,11 @@ public abstract class AbstractIntroduceGroupByCombinerRule extends AbstractIntro Object v2 = gbyOp.getAnnotations().get(OperatorAnnotations.USE_EXTERNAL_GROUP_BY); newGbyOp.getAnnotations().put(OperatorAnnotations.USE_EXTERNAL_GROUP_BY, v2); - List<LogicalVariable> propagatedVars = new LinkedList<LogicalVariable>(); - VariableUtilities.getProducedVariables(newGbyOp, propagatedVars); - Set<LogicalVariable> freeVars = new HashSet<LogicalVariable>(); OperatorPropertiesUtil.getFreeVariablesInSubplans(gbyOp, freeVars); for (LogicalVariable var : freeVars) { - if (!propagatedVars.contains(var)) { + if (!newGbyLiveVars.contains(var)) { LogicalVariable newDecorVar = context.newVar(); VariableReferenceExpression varRef = new VariableReferenceExpression(var); varRef.setSourceLocation(gbyOp.getSourceLocation()); @@ -435,8 +434,8 @@ public abstract class AbstractIntroduceGroupByCombinerRule extends AbstractIntro * The optimization context. * @param nestedGby * The nested gby operator in the global gby operator's subplan. - * @param firstAggVar - * The first aggregation variable produced by the combiner gby. + * @param aggregateVarsProducedByCombiner + * The aggregation variables produced by the combiner gby. */ protected abstract void processNullTest(IOptimizationContext context, GroupByOperator nestedGby, List<LogicalVariable> aggregateVarsProducedByCombiner);