Yingyi Bu has submitted this change and it was merged. Change subject: ASTERIXDB-1626: performance improvement for multiple global aggregates. ......................................................................
ASTERIXDB-1626: performance improvement for multiple global aggregates. - add a groupAll option for preclustered group-by. - the listify function is removed in multiple global aggregate plans. Change-Id: I85bb47748950cc909ddbd9720f613e0b8956d320 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1185 Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Till Westmann <ti...@apache.org> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> --- M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlPlusExpressionToPlanTranslator.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java A asterixdb/asterix-app/src/test/resources/optimizerts/queries/aggregate/query-ASTERIXDB-1626-2.sqlpp A asterixdb/asterix-app/src/test/resources/optimizerts/queries/aggregate/query-ASTERIXDB-1626-3.sqlpp A asterixdb/asterix-app/src/test/resources/optimizerts/queries/aggregate/query-ASTERIXDB-1626.sqlpp A asterixdb/asterix-app/src/test/resources/optimizerts/results/aggregate/query-ASTERIXDB-1626-2.plan A asterixdb/asterix-app/src/test/resources/optimizerts/results/aggregate/query-ASTERIXDB-1626-3.plan A asterixdb/asterix-app/src/test/resources/optimizerts/results/aggregate/query-ASTERIXDB-1626.plan A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.3.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.4.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.3.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.4.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.1.adm A asterixdb/asterix-app/src/test/resources/runtimets/results/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/clause/GroupbyClause.java M asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGlobalAggregationSugarVisitor.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/CardinalityInferenceVisitor.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalOperatorDeepCopyWithNewVariablesVisitor.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java M hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java M hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EliminateGroupByEmptyKeyRule.java M hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushGroupByIntoSortRule.java M hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java M hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java M hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/preclustered/PreclusteredGroupOperatorDescriptor.java M hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/preclustered/PreclusteredGroupOperatorNodePushable.java M hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/preclustered/PreclusteredGroupWriter.java 40 files changed, 567 insertions(+), 176 deletions(-) Approvals: Till Westmann: Looks good to me, approved Jenkins: Verified; Verified Objections: Jenkins: Violations found diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java index 9382b2d..677b9a7 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java @@ -171,7 +171,7 @@ columnList.add(varRef.getVariableReference()); } } - op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList)); + op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList, gby.isGroupAll())); } } } else if (((AbstractLogicalOperator) (r0.getValue())).getOperatorTag() @@ -185,7 +185,7 @@ columnList.add(varRef.getVariableReference()); } } - op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList)); + op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList, gby.isGroupAll())); } else { throw new AlgebricksException("Unsupported nested operator within a group-by: " + ((AbstractLogicalOperator) (r0.getValue())).getOperatorTag().name()); diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java index 8e5fa79..143cf0b 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java @@ -313,9 +313,9 @@ // where the keyVarsToEnforce forms a candidate key which can // uniquely identify a tuple out of the nested-tuple-source. LogicalVariable newVar = context.newVar(); - gbyOp.getGroupByList().add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(newVar, - new MutableObject<ILogicalExpression>(new VariableReferenceExpression(keyVar)))); - keyVarNewVarPairs.add(new Pair<LogicalVariable, LogicalVariable>(keyVar, newVar)); + gbyOp.getGroupByList() + .add(new Pair<>(newVar, new MutableObject<>(new VariableReferenceExpression(keyVar)))); + keyVarNewVarPairs.add(new Pair<>(keyVar, newVar)); } // Creates an aggregate operator doing LISTIFY, as the root of the @@ -326,31 +326,31 @@ List<Mutable<ILogicalExpression>> aggArgList = new ArrayList<>(); aggVarList.add(aggVar); // Creates an aggregation function expression. - aggArgList.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(recordVar))); + aggArgList.add(new MutableObject<>(new VariableReferenceExpression(recordVar))); ILogicalExpression aggExpr = new AggregateFunctionCallExpression( FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.LISTIFY), false, aggArgList); - aggExprList.add(new MutableObject<ILogicalExpression>(aggExpr)); + aggExprList.add(new MutableObject<>(aggExpr)); AggregateOperator aggOp = new AggregateOperator(aggVarList, aggExprList); // Adds the original limit operator as the input operator to the added // aggregate operator. - aggOp.getInputs().add(new MutableObject<ILogicalOperator>(op)); + aggOp.getInputs().add(new MutableObject<>(op)); op.getInputs().clear(); ILogicalOperator currentOp = op; if (!orderingExprs.isEmpty()) { OrderOperator orderOp = new OrderOperator(cloneOrderingExpression(orderingExprs)); - op.getInputs().add(new MutableObject<ILogicalOperator>(orderOp)); + op.getInputs().add(new MutableObject<>(orderOp)); currentOp = orderOp; } // Adds a nested tuple source operator as the input operator to the // limit operator. NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(gbyOp)); - currentOp.getInputs().add(new MutableObject<ILogicalOperator>(nts)); + currentOp.getInputs().add(new MutableObject<>(nts)); // Sets the root of the added nested plan to the aggregate operator. ILogicalPlan nestedPlan = new ALogicalPlanImpl(); - nestedPlan.getRoots().add(new MutableObject<ILogicalOperator>(aggOp)); + nestedPlan.getRoots().add(new MutableObject<>(aggOp)); // Sets the nested plan for the added group-by operator. gbyOp.getNestedPlans().add(nestedPlan); @@ -359,7 +359,7 @@ for (Pair<LogicalVariable, LogicalVariable> keyVarNewVar : keyVarNewVarPairs) { updateInputToOutputVarMapping(keyVarNewVar.first, keyVarNewVar.second, false); } - return new Pair<ILogicalOperator, LogicalVariable>(gbyOp, aggVar); + return new Pair<>(gbyOp, aggVar); } private Pair<ILogicalOperator, LogicalVariable> createUnnestForAggregatedList(LogicalVariable aggVar) { @@ -645,20 +645,20 @@ // where the keyVarsToEnforce forms a candidate key which can // uniquely identify a tuple out of the nested-tuple-source. LogicalVariable newVar = context.newVar(); - gbyOp.getGroupByList().add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(newVar, - new MutableObject<ILogicalExpression>(new VariableReferenceExpression(keyVar)))); + gbyOp.getGroupByList() + .add(new Pair<>(newVar, new MutableObject<>(new VariableReferenceExpression(keyVar)))); updateInputToOutputVarMapping(keyVar, newVar, false); } ILogicalOperator inputOp = op.getInputs().get(0).getValue(); - gbyOp.getInputs().add(new MutableObject<ILogicalOperator>(inputOp)); + gbyOp.getInputs().add(new MutableObject<>(inputOp)); NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(gbyOp)); op.getInputs().clear(); - op.getInputs().add(new MutableObject<ILogicalOperator>(nts)); + op.getInputs().add(new MutableObject<>(nts)); ILogicalPlan nestedPlan = new ALogicalPlanImpl(); - nestedPlan.getRoots().add(new MutableObject<ILogicalOperator>(op)); + nestedPlan.getRoots().add(new MutableObject<>(op)); gbyOp.getNestedPlans().add(nestedPlan); OperatorManipulationUtil.computeTypeEnvironmentBottomUp(gbyOp, context); diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlPlusExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlPlusExpressionToPlanTranslator.java index 05bd343..45e66cb 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlPlusExpressionToPlanTranslator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlPlusExpressionToPlanTranslator.java @@ -565,7 +565,7 @@ // the one bound by the aggregation op. context.setVar(entry.getValue(), aggVar); } - + gOp.setGroupAll(gc.isGroupAll()); gOp.getAnnotations().put(OperatorAnnotations.USE_HASH_GROUP_BY, gc.hasHashGroupByHint()); return new Pair<ILogicalOperator, LogicalVariable>(gOp, null); } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java index b73ef82..3a0629d 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java @@ -852,94 +852,46 @@ groupVarAssignOp.getInputs().add(topOp); topOp = new MutableObject<>(groupVarAssignOp); } - if (gc.isGroupAll()) { - List<LogicalVariable> aggVars = new ArrayList<>(); - List<Mutable<ILogicalExpression>> aggFuncs = new ArrayList<>(); - // A global aggregation can still have a decoration variable list which are used for propagate - // outer-scope variables. Example query: - // asterixdb/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/q09 - for (GbyVariableExpressionPair ve : gc.getDecorPairList()) { - VariableExpr vexpr = ve.getVar(); - LogicalVariable decorVar = vexpr == null ? context.newVar() : context.newVar(vexpr); - Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = langExprToAlgExpression(ve.getExpr(), topOp); - topOp = eo.second; - List<Mutable<ILogicalExpression>> flArgs = new ArrayList<>(); - flArgs.add(new MutableObject<>(eo.first)); - // Calls the first-element aggregate function on a decoration variable to make sure the value - // is propagated through a global aggregation. - AggregateFunctionCallExpression firstElementAgg = AsterixBuiltinFunctions - .makeAggregateFunctionExpression(AsterixBuiltinFunctions.FIRST_ELEMENT, flArgs); - aggVars.add(decorVar); - aggFuncs.add(new MutableObject<>(firstElementAgg)); - } - for (Entry<Expression, VariableExpr> entry : gc.getWithVarMap().entrySet()) { - Pair<ILogicalExpression, Mutable<ILogicalOperator>> listifyInput = langExprToAlgExpression( - entry.getKey(), topOp); - topOp = listifyInput.second; - List<Mutable<ILogicalExpression>> flArgs = new ArrayList<>(); - flArgs.add(new MutableObject<>(listifyInput.first)); - AggregateFunctionCallExpression fListify = AsterixBuiltinFunctions - .makeAggregateFunctionExpression(AsterixBuiltinFunctions.LISTIFY, flArgs); - LogicalVariable aggVar = context.newVar(); - aggVars.add(aggVar); - aggFuncs.add(new MutableObject<>(fListify)); - // Hide the variable that was part of the "with", replacing it with - // the one bound by the aggregation op. - context.setVar(entry.getValue(), aggVar); - } - AggregateOperator aggOp = new AggregateOperator(aggVars, aggFuncs); - aggOp.getInputs().add(topOp); - return new Pair<>(aggOp, null); - } else { - GroupByOperator gOp = new GroupByOperator(); - for (GbyVariableExpressionPair ve : gc.getGbyPairList()) { - LogicalVariable v; - VariableExpr vexpr = ve.getVar(); - if (vexpr != null) { - v = context.newVar(vexpr); - } else { - v = context.newVar(); - } - Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = langExprToAlgExpression(ve.getExpr(), topOp); - gOp.addGbyExpression(v, eo.first); - topOp = eo.second; - } - for (GbyVariableExpressionPair ve : gc.getDecorPairList()) { - LogicalVariable v; - VariableExpr vexpr = ve.getVar(); - if (vexpr != null) { - v = context.newVar(vexpr); - } else { - v = context.newVar(); - } - Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = langExprToAlgExpression(ve.getExpr(), topOp); - gOp.addDecorExpression(v, eo.first); - topOp = eo.second; - } - gOp.getInputs().add(topOp); - for (Entry<Expression, VariableExpr> entry : gc.getWithVarMap().entrySet()) { - Pair<ILogicalExpression, Mutable<ILogicalOperator>> listifyInput = langExprToAlgExpression( - entry.getKey(), new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(gOp)))); - List<Mutable<ILogicalExpression>> flArgs = new ArrayList<>(1); - flArgs.add(new MutableObject<>(listifyInput.first)); - AggregateFunctionCallExpression fListify = AsterixBuiltinFunctions - .makeAggregateFunctionExpression(AsterixBuiltinFunctions.LISTIFY, flArgs); - LogicalVariable aggVar = context.newVar(); - AggregateOperator agg = new AggregateOperator(mkSingletonArrayList(aggVar), - mkSingletonArrayList(new MutableObject<>(fListify))); - - agg.getInputs().add(listifyInput.second); - ILogicalPlan plan = new ALogicalPlanImpl(new MutableObject<>(agg)); - gOp.getNestedPlans().add(plan); - // Hide the variable that was part of the "with", replacing it with - // the one bound by the aggregation op. - context.setVar(entry.getValue(), aggVar); - } - gOp.getAnnotations().put(OperatorAnnotations.USE_HASH_GROUP_BY, gc.hasHashGroupByHint()); - return new Pair<>(gOp, null); + GroupByOperator gOp = new GroupByOperator(); + for (GbyVariableExpressionPair ve : gc.getGbyPairList()) { + VariableExpr vexpr = ve.getVar(); + LogicalVariable v = vexpr == null ? context.newVar() : context.newVar(vexpr); + Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = langExprToAlgExpression(ve.getExpr(), topOp); + gOp.addGbyExpression(v, eo.first); + topOp = eo.second; + } + for (GbyVariableExpressionPair ve : gc.getDecorPairList()) { + VariableExpr vexpr = ve.getVar(); + LogicalVariable v = vexpr == null ? context.newVar() : context.newVar(vexpr); + Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = langExprToAlgExpression(ve.getExpr(), topOp); + gOp.addDecorExpression(v, eo.first); + topOp = eo.second; } + gOp.getInputs().add(topOp); + for (Entry<Expression, VariableExpr> entry : gc.getWithVarMap().entrySet()) { + Pair<ILogicalExpression, Mutable<ILogicalOperator>> listifyInput = langExprToAlgExpression( + entry.getKey(), new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(gOp)))); + List<Mutable<ILogicalExpression>> flArgs = new ArrayList<>(1); + flArgs.add(new MutableObject<>(listifyInput.first)); + AggregateFunctionCallExpression fListify = AsterixBuiltinFunctions + .makeAggregateFunctionExpression(AsterixBuiltinFunctions.LISTIFY, flArgs); + LogicalVariable aggVar = context.newVar(); + AggregateOperator agg = new AggregateOperator(mkSingletonArrayList(aggVar), + mkSingletonArrayList(new MutableObject<>(fListify))); + + agg.getInputs().add(listifyInput.second); + + ILogicalPlan plan = new ALogicalPlanImpl(new MutableObject<>(agg)); + gOp.getNestedPlans().add(plan); + // Hide the variable that was part of the "with", replacing it with + // the one bound by the aggregation op. + context.setVar(entry.getValue(), aggVar); + } + gOp.setGroupAll(gc.isGroupAll()); + gOp.getAnnotations().put(OperatorAnnotations.USE_HASH_GROUP_BY, gc.hasHashGroupByHint()); + return new Pair<>(gOp, null); } @Override diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/aggregate/query-ASTERIXDB-1626-2.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/aggregate/query-ASTERIXDB-1626-2.sqlpp new file mode 100644 index 0000000..c22d86d --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/aggregate/query-ASTERIXDB-1626-2.sqlpp @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +drop dataverse TinySocial if exists; +create dataverse TinySocial; + +use TinySocial; + + +create type TinySocial.FacebookUserType as + open { + id : int64 +} + +create dataset FacebookUsers(FacebookUserType) primary key id; + +SELECT count(`friend-ids`) * avg(coll_count(`friend-ids`)) AS total +FROM FacebookUsers; diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/aggregate/query-ASTERIXDB-1626-3.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/aggregate/query-ASTERIXDB-1626-3.sqlpp new file mode 100644 index 0000000..cbb9823 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/aggregate/query-ASTERIXDB-1626-3.sqlpp @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +drop dataverse TinySocial if exists; +create dataverse TinySocial; + +use TinySocial; + + +create type TinySocial.FacebookUserType as + open { + id : int64 +} + +create dataset FacebookUsers(FacebookUserType) primary key id; + +// The plan uses a group-by operator with the groupAll flag be true. +WITH t AS 1 +SELECT count(`friend-ids`) * avg(coll_count(`friend-ids`)) AS total, t +FROM FacebookUsers; diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/aggregate/query-ASTERIXDB-1626.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/aggregate/query-ASTERIXDB-1626.sqlpp new file mode 100644 index 0000000..37ad467 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/aggregate/query-ASTERIXDB-1626.sqlpp @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +drop dataverse TinySocial if exists; +create dataverse TinySocial; + +use TinySocial; + + +create type TinySocial.FacebookUserType as + open { + id : int64 +} + +create dataset FacebookUsers(FacebookUserType) primary key id; + +SELECT count(`friend-ids`), avg(coll_count(`friend-ids`)) +FROM FacebookUsers; diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/aggregate/query-ASTERIXDB-1626-2.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/aggregate/query-ASTERIXDB-1626-2.plan new file mode 100644 index 0000000..8ed84bc --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/aggregate/query-ASTERIXDB-1626-2.plan @@ -0,0 +1,15 @@ +-- DISTRIBUTE_RESULT |UNPARTITIONED| + -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| + -- STREAM_PROJECT |UNPARTITIONED| + -- ASSIGN |UNPARTITIONED| + -- AGGREGATE |UNPARTITIONED| + -- RANDOM_MERGE_EXCHANGE |PARTITIONED| + -- AGGREGATE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/aggregate/query-ASTERIXDB-1626-3.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/aggregate/query-ASTERIXDB-1626-3.plan new file mode 100644 index 0000000..b5c2be0 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/aggregate/query-ASTERIXDB-1626-3.plan @@ -0,0 +1,24 @@ +-- DISTRIBUTE_RESULT |UNPARTITIONED| + -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| + -- STREAM_PROJECT |UNPARTITIONED| + -- ASSIGN |UNPARTITIONED| + -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| + -- PRE_CLUSTERED_GROUP_BY(ALL)[] |UNPARTITIONED| + { + -- AGGREGATE |LOCAL| + -- NESTED_TUPLE_SOURCE |LOCAL| + } + -- RANDOM_MERGE_EXCHANGE |PARTITIONED| + -- PRE_CLUSTERED_GROUP_BY(ALL)[] |PARTITIONED| + { + -- AGGREGATE |LOCAL| + -- NESTED_TUPLE_SOURCE |LOCAL| + } + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/aggregate/query-ASTERIXDB-1626.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/aggregate/query-ASTERIXDB-1626.plan new file mode 100644 index 0000000..8ed84bc --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/aggregate/query-ASTERIXDB-1626.plan @@ -0,0 +1,15 @@ +-- DISTRIBUTE_RESULT |UNPARTITIONED| + -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| + -- STREAM_PROJECT |UNPARTITIONED| + -- ASSIGN |UNPARTITIONED| + -- AGGREGATE |UNPARTITIONED| + -- RANDOM_MERGE_EXCHANGE |PARTITIONED| + -- AGGREGATE |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ASSIGN |PARTITIONED| + -- STREAM_PROJECT |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- DATASOURCE_SCAN |PARTITIONED| + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- EMPTY_TUPLE_SOURCE |PARTITIONED| diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.1.ddl.sqlpp new file mode 100644 index 0000000..78ce34a --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.1.ddl.sqlpp @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse TinySocial if exists; +create dataverse TinySocial; + +use TinySocial; + +create type TinySocial.FacebookUserType as + open { + id : int64 +} + +create dataset FacebookUsers(FacebookUserType) primary key id; + diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.2.update.sqlpp new file mode 100644 index 0000000..9f4f824 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.2.update.sqlpp @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use TinySocial; + + +load dataset FacebookUsers using localfs ((`path`=`asterix_nc1://data/tinysocial/fbu.adm`),(`format`=`adm`)); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.3.query.sqlpp new file mode 100644 index 0000000..15af505 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.3.query.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE TinySocial; + +WITH t AS 1 +SELECT count(`friend-ids`), avg(coll_count(`friend-ids`)), t +FROM FacebookUsers; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.4.ddl.sqlpp new file mode 100644 index 0000000..3509f58 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.4.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse TinySocial if exists; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.1.ddl.sqlpp new file mode 100644 index 0000000..78ce34a --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.1.ddl.sqlpp @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse TinySocial if exists; +create dataverse TinySocial; + +use TinySocial; + +create type TinySocial.FacebookUserType as + open { + id : int64 +} + +create dataset FacebookUsers(FacebookUserType) primary key id; + diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.2.update.sqlpp new file mode 100644 index 0000000..9f4f824 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.2.update.sqlpp @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use TinySocial; + + +load dataset FacebookUsers using localfs ((`path`=`asterix_nc1://data/tinysocial/fbu.adm`),(`format`=`adm`)); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.3.query.sqlpp new file mode 100644 index 0000000..c72c3c8 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.3.query.sqlpp @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE TinySocial; + +SELECT count(`friend-ids`), avg(coll_count(`friend-ids`)) +FROM FacebookUsers; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.4.ddl.sqlpp new file mode 100644 index 0000000..3509f58 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.4.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse TinySocial if exists; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.1.adm new file mode 100644 index 0000000..5a6f130 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/global-aggregate/query-ASTERIXDB-1626-2/query-ASTERIXDB-1626-2.1.adm @@ -0,0 +1 @@ +{ "$1": 10, "$2": 2.4, "t": 1 } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.1.adm new file mode 100644 index 0000000..b14dc70 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/global-aggregate/query-ASTERIXDB-1626/query-ASTERIXDB-1626.1.adm @@ -0,0 +1 @@ +{ "$1": 10, "$2": 2.4 } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index c19593b..b6fe6f8 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -2480,6 +2480,16 @@ <output-dir compare="Text">query-ASTERIXDB-159</output-dir> </compilation-unit> </test-case> + <test-case FilePath="global-aggregate"> + <compilation-unit name="query-ASTERIXDB-1626"> + <output-dir compare="Text">query-ASTERIXDB-1626</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="global-aggregate"> + <compilation-unit name="query-ASTERIXDB-1626-2"> + <output-dir compare="Text">query-ASTERIXDB-1626-2</output-dir> + </compilation-unit> + </test-case> </test-group> <test-group name="group-by"> <test-case FilePath="group-by"> diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/clause/GroupbyClause.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/clause/GroupbyClause.java index 0f75839..7dacd39 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/clause/GroupbyClause.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/clause/GroupbyClause.java @@ -39,8 +39,8 @@ private Map<Expression, VariableExpr> withVarMap; private VariableExpr groupVar; private List<Pair<Expression, Identifier>> groupFieldList = new ArrayList<>(); - private boolean hashGroupByHint; - private boolean groupAll; + private boolean hashGroupByHint = false; + private boolean groupAll = false; public GroupbyClause() { // Default constructor. diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGlobalAggregationSugarVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGlobalAggregationSugarVisitor.java index 6308d7f..9430e4f 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGlobalAggregationSugarVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGlobalAggregationSugarVisitor.java @@ -49,7 +49,6 @@ if (addImplicitGby) { // Adds an implicit group-by clause for SQL-92 global aggregate. List<GbyVariableExpressionPair> gbyPairList = new ArrayList<>(); - gbyPairList.add(new GbyVariableExpressionPair(null, new LiteralExpr(new IntegerLiteral(1)))); List<GbyVariableExpressionPair> decorPairList = new ArrayList<>(); GroupbyClause gbyClause = new GroupbyClause(gbyPairList, decorPairList, new HashMap<>(), null, null, false, true); diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java index 5b0fd14..aa694b8 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java @@ -24,7 +24,6 @@ import org.apache.commons.lang3.mutable.Mutable; import org.apache.commons.lang3.mutable.MutableObject; - import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; @@ -53,27 +52,36 @@ // In decorList, if the variable (first member of the pair) is null, the // second member of the pair is variable reference which is propagated. + private boolean groupAll = false; + private boolean global = true; + public GroupByOperator() { super(); - gByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>(); - decorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>(); + gByList = new ArrayList<>(); + decorList = new ArrayList<>(); } public GroupByOperator(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList, List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList, List<ILogicalPlan> nestedPlans) { + this(groupByList, decorList, nestedPlans, false); + } + + public GroupByOperator(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList, + List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList, List<ILogicalPlan> nestedPlans, + boolean groupAll) { super(nestedPlans); this.decorList = decorList; this.gByList = groupByList; + this.groupAll = groupAll; + checkGroupAll(groupAll); } public void addGbyExpression(LogicalVariable variable, ILogicalExpression expression) { - this.gByList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(variable, - new MutableObject<ILogicalExpression>(expression))); + this.gByList.add(new Pair<>(variable, new MutableObject<>(expression))); } public void addDecorExpression(LogicalVariable variable, ILogicalExpression expression) { - this.decorList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(variable, - new MutableObject<ILogicalExpression>(expression))); + this.decorList.add(new Pair<>(variable, new MutableObject<>(expression))); } @Override @@ -85,30 +93,14 @@ return gByList; } - public String gByListToString() { - return veListToString(gByList); - } - - public String decorListToString() { - return veListToString(decorList); - } - public List<LogicalVariable> getGbyVarList() { - List<LogicalVariable> varList = new ArrayList<LogicalVariable>(gByList.size()); + List<LogicalVariable> varList = new ArrayList<>(gByList.size()); for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gByList) { ILogicalExpression expr = ve.second.getValue(); if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) { VariableReferenceExpression v = (VariableReferenceExpression) expr; varList.add(v.getVariableReference()); } - } - return varList; - } - - public List<LogicalVariable> getProducedGbyVarList() { - List<LogicalVariable> varList = new ArrayList<LogicalVariable>(gByList.size()); - for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gByList) { - varList.add(ve.first); } return varList; } @@ -146,7 +138,6 @@ @Override public void getProducedVariablesExceptNestedPlans(Collection<LogicalVariable> vars) { - // super.getProducedVariables(vars); for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) { if (p.first != null) { vars.add(p.first); @@ -167,7 +158,6 @@ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : decorList) { g.second.getValue().getUsedVariables(vars); } - // super.getUsedVariables(vars); } @Override @@ -285,4 +275,30 @@ } return env; } + + public boolean isGroupAll() { + return groupAll; + } + + public void setGroupAll(boolean groupAll) { + this.groupAll = groupAll; + checkGroupAll(groupAll); + } + + public boolean isGlobal() { + return global; + } + + public void setGlobal(boolean global) { + this.global = global; + } + + // The groupAll flag can only be set if group by columns are empty. + private void checkGroupAll(boolean groupAll) { + if (groupAll && !gByList.isEmpty()) { + throw new IllegalStateException( + "Conflicting parameters for GROUP BY: there should be no GROUP BY keys " + + "when the GROUP ALL flag is set to true"); + } + } } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/CardinalityInferenceVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/CardinalityInferenceVisitor.java index ae98d05..b999493 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/CardinalityInferenceVisitor.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/CardinalityInferenceVisitor.java @@ -107,6 +107,9 @@ @Override public Long visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException { + if (op.isGroupAll()) { + return ONE; + } ILogicalOperator inputOp = op.getInputs().get(0).getValue(); long inputCardinality = inputOp.accept(this, arg); List<LogicalVariable> gbyVar = op.getGbyVarList(); diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalOperatorDeepCopyWithNewVariablesVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalOperatorDeepCopyWithNewVariablesVisitor.java index 54acf2f..4241d84 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalOperatorDeepCopyWithNewVariablesVisitor.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalOperatorDeepCopyWithNewVariablesVisitor.java @@ -328,7 +328,7 @@ op.getDecorList()); List<ILogicalPlan> nestedPlansCopy = new ArrayList<ILogicalPlan>(); - GroupByOperator opCopy = new GroupByOperator(groupByListCopy, decorListCopy, nestedPlansCopy); + GroupByOperator opCopy = new GroupByOperator(groupByListCopy, decorListCopy, nestedPlansCopy, op.isGroupAll()); deepCopyInputsAnnotationsAndExecutionMode(op, arg, opCopy); deepCopyPlanList(op.getNestedPlans(), nestedPlansCopy, opCopy); return opCopy; diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java index 99123b3..fde6a28 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java @@ -52,7 +52,6 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder; import org.apache.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator; @@ -67,6 +66,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.WriteOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder; import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil; import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor; @@ -108,7 +108,7 @@ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : op.getDecorList()) { decoList.add(new Pair<>(pair.first, deepCopyExpressionRef(pair.second))); } - GroupByOperator gbyOp = new GroupByOperator(groupByList, decoList, newSubplans); + GroupByOperator gbyOp = new GroupByOperator(groupByList, decoList, newSubplans, op.isGroupAll()); for (ILogicalPlan plan : op.getNestedPlans()) { newSubplans.add(OperatorManipulationUtil.deepCopy(plan, gbyOp)); } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java index c0bdcb4..9c06d2c 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java @@ -28,7 +28,6 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.ListSet; import org.apache.hyracks.algebricks.common.utils.Pair; -import org.apache.hyracks.algebricks.core.algebra.base.EquivalenceClass; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan; @@ -39,11 +38,10 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode; import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode; import org.apache.hyracks.algebricks.core.algebra.properties.FunctionalDependency; import org.apache.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty; -import org.apache.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType; import org.apache.hyracks.algebricks.core.algebra.properties.IPartitioningProperty; import org.apache.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator; import org.apache.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector; @@ -54,6 +52,7 @@ import org.apache.hyracks.algebricks.core.algebra.properties.PropertiesUtil; import org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector; import org.apache.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty; +import org.apache.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType; import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil; public abstract class AbstractPreclusteredGroupByPOperator extends AbstractPhysicalOperator { @@ -103,12 +102,22 @@ @Override public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op, IPhysicalPropertiesVector reqdByParent, IOptimizationContext context) { + GroupByOperator gby = (GroupByOperator) op; StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1]; + if (gby.isGroupAll() && gby.isGlobal()) { + if (op.getExecutionMode() == ExecutionMode.UNPARTITIONED) { + pv[0] = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, null); + return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION); + } else { + return emptyUnaryRequirements(); + } + } + List<ILocalStructuralProperty> localProps = new ArrayList<>(); Set<LogicalVariable> gbvars = new ListSet<>(columnList); - LocalGroupingProperty groupProp = new LocalGroupingProperty(gbvars, new ArrayList<LogicalVariable>(columnList)); + LocalGroupingProperty groupProp = new LocalGroupingProperty(gbvars, new ArrayList<>(columnList)); - GroupByOperator gby = (GroupByOperator) op; + boolean goon = true; for (ILogicalPlan p : gby.getNestedPlans()) { // try to propagate secondary order requirements from nested @@ -186,7 +195,7 @@ fdList.add(new FunctionalDependency(hd, tl)); } if (allOk && PropertiesUtil.matchLocalProperties(localProps, props, - new HashMap<LogicalVariable, EquivalenceClass>(), fdList)) { + new HashMap<>(), fdList)) { localProps = props; } } @@ -195,7 +204,7 @@ IPartitioningProperty pp = null; AbstractLogicalOperator aop = (AbstractLogicalOperator) op; if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) { - pp = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(columnList), + pp = new UnorderedPartitionedProperty(new ListSet<>(columnList), context.getComputationNodeDomain()); } pv[0] = new StructuralPropertiesVector(pp, localProps); diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java index 3242fa0..f16f49a 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java @@ -22,7 +22,6 @@ import java.util.List; import org.apache.commons.lang3.mutable.Mutable; - import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; @@ -35,7 +34,6 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema; -import org.apache.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty; import org.apache.hyracks.algebricks.core.algebra.properties.IPartitioningProperty; import org.apache.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator; import org.apache.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector; @@ -63,10 +61,10 @@ ILogicalOperator op2 = op.getInputs().get(0).getValue(); if (aggOp.getExecutionMode() != AbstractLogicalOperator.ExecutionMode.UNPARTITIONED) { deliveredProperties = new StructuralPropertiesVector(op2.getDeliveredPhysicalProperties() - .getPartitioningProperty(), new ArrayList<ILocalStructuralProperty>()); + .getPartitioningProperty(), new ArrayList<>()); } else { deliveredProperties = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, - new ArrayList<ILocalStructuralProperty>()); + new ArrayList<>()); } } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java index 36bd73d..e0df2f9 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java @@ -21,7 +21,6 @@ import java.util.List; import org.apache.commons.lang3.mutable.Mutable; - import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder; @@ -48,8 +47,11 @@ public class PreclusteredGroupByPOperator extends AbstractPreclusteredGroupByPOperator { - public PreclusteredGroupByPOperator(List<LogicalVariable> columnList) { + private final boolean groupAll; + + public PreclusteredGroupByPOperator(List<LogicalVariable> columnList, boolean groupAll) { super(columnList); + this.groupAll = groupAll; } @Override @@ -84,7 +86,8 @@ AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], gby, opSchema, context); IAggregatorDescriptorFactory aggregatorFactory; - if (((AbstractLogicalOperator) (gby.getNestedPlans().get(0).getRoots().get(0).getValue())).getOperatorTag() == LogicalOperatorTag.RUNNINGAGGREGATE) { + if (gby.getNestedPlans().get(0).getRoots().get(0).getValue() + .getOperatorTag() == LogicalOperatorTag.RUNNINGAGGREGATE) { aggregatorFactory = new NestedPlansRunningAggregatorFactory(subplans, keys, fdColumns); } else { aggregatorFactory = new NestedPlansAccumulatingAggregatorFactory(subplans, keys, fdColumns); @@ -97,7 +100,7 @@ context); PreclusteredGroupOperatorDescriptor opDesc = new PreclusteredGroupOperatorDescriptor(spec, keys, - comparatorFactories, aggregatorFactory, recordDescriptor); + comparatorFactories, aggregatorFactory, recordDescriptor, groupAll); contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc); @@ -105,4 +108,9 @@ builder.contributeGraphEdge(src, 0, op, 0); } + @Override + public String toString() { + return getOperatorTag().toString() + (groupAll ? "(ALL)" : "") + columnList; + } + } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java index 96d06c4..7e83880 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java @@ -41,7 +41,6 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator.Kind; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator; @@ -64,6 +63,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.WriteOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator.Kind; import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor; import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor; @@ -131,7 +131,7 @@ @Override public Void visitGroupByOperator(GroupByOperator op, Integer indent) throws AlgebricksException { - addIndent(indent).append("group by ("); + addIndent(indent).append("group by" + (op.isGroupAll() ? " (all)" : "") + " ("); pprintVeList(op.getGroupByList(), indent); buffer.append(") decor ("); pprintVeList(op.getDecorList(), indent); diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java index 9b8258c..8a8fe6d 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java @@ -34,6 +34,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator; @@ -120,6 +121,13 @@ forceUnpartitioned = true; } } + if (op.getOperatorTag() == LogicalOperatorTag.GROUP) { + GroupByOperator gbyOp = (GroupByOperator) op; + if (gbyOp.isGroupAll() && gbyOp.isGlobal()) { + op.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED); + forceUnpartitioned = true; + } + } for (Mutable<ILogicalOperator> i : op.getInputs()) { boolean exit = false; diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java index 4d2fc30..4ef1cd5 100644 --- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java +++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java @@ -40,11 +40,11 @@ import org.apache.hyracks.algebricks.core.algebra.base.OperatorAnnotations; import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode; import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.IsomorphismUtilities; import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities; import org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl; @@ -64,7 +64,9 @@ return false; } GroupByOperator gbyOp = (GroupByOperator) op; - if (gbyOp.getExecutionMode() != ExecutionMode.PARTITIONED) { + ExecutionMode executionMode = gbyOp.getExecutionMode(); + if (executionMode != ExecutionMode.PARTITIONED + && !(executionMode == ExecutionMode.UNPARTITIONED && gbyOp.isGroupAll())) { return false; } @@ -195,6 +197,11 @@ newGbyOp.addGbyExpression(replGbyList.get(i), new VariableReferenceExpression(newOpGbyList.get(i))); VariableUtilities.substituteVariables(gbyOp, newOpGbyList.get(i), replGbyList.get(i), false, context); } + + // Sets the global flag to be false. + newGbyOp.setGlobal(false); + // Sets the group all flag. + newGbyOp.setGroupAll(gbyOp.isGroupAll()); return newGbyOp; } diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EliminateGroupByEmptyKeyRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EliminateGroupByEmptyKeyRule.java index 8a91cfc..19dc21e 100644 --- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EliminateGroupByEmptyKeyRule.java +++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EliminateGroupByEmptyKeyRule.java @@ -55,6 +55,10 @@ return false; } GroupByOperator groupOp = (GroupByOperator) op; + // Only groupAll has equivalent semantics to aggregate. + if (!groupOp.isGroupAll()) { + return false; + } List<LogicalVariable> groupVars = groupOp.getGbyVarList(); List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList = groupOp.getDecorList(); if (!groupVars.isEmpty() || !decorList.isEmpty()) { diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushGroupByIntoSortRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushGroupByIntoSortRule.java index 2ffc8d6..576bd62 100644 --- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushGroupByIntoSortRule.java +++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushGroupByIntoSortRule.java @@ -23,7 +23,6 @@ import org.apache.commons.lang3.mutable.Mutable; import org.apache.commons.lang3.mutable.MutableObject; - import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; @@ -101,10 +100,11 @@ } //replace preclustered gby with sort gby - op.setPhysicalOperator(new SortGroupByPOperator(groupByOperator.getGroupByList(), context - .getPhysicalOptimizationConfig().getMaxFramesExternalGroupBy(), sortPhysicalOperator - .getSortColumns())); - + if (!groupByOperator.isGroupAll()) { + op.setPhysicalOperator(new SortGroupByPOperator(groupByOperator.getGroupByList(), + context.getPhysicalOptimizationConfig().getMaxFramesExternalGroupBy(), + sortPhysicalOperator.getSortColumns())); + } // remove the stable sort operator op.getInputs().clear(); op.getInputs().addAll(op2.getInputs()); diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java index ddfb331..0c09fc0 100644 --- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java +++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java @@ -187,7 +187,7 @@ } } if (topLevelOp) { - op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList)); + op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList, gby.isGroupAll())); } else { op.setPhysicalOperator(new MicroPreclusteredGroupByPOperator(columnList)); } diff --git a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java index 974a079..688e819 100644 --- a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java +++ b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java @@ -108,10 +108,11 @@ byte[] data = tb.getByteArray(); int[] fieldEnds = tb.getFieldEndOffsets(); int start = 0; - int offset = 0; + int offset; for (int i = 0; i < fieldEnds.length; i++) { - if (i > 0) + if (i > 0) { start = fieldEnds[i - 1]; + } offset = fieldEnds[i] - start; tupleBuilder.addField(data, start, offset); } diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/preclustered/PreclusteredGroupOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/preclustered/PreclusteredGroupOperatorDescriptor.java index 8df0002..b45879c 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/preclustered/PreclusteredGroupOperatorDescriptor.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/preclustered/PreclusteredGroupOperatorDescriptor.java @@ -29,20 +29,28 @@ import org.apache.hyracks.dataflow.std.group.IAggregatorDescriptorFactory; public class PreclusteredGroupOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor { + private static final long serialVersionUID = 1L; + private final int[] groupFields; private final IBinaryComparatorFactory[] comparatorFactories; private final IAggregatorDescriptorFactory aggregatorFactory; - - private static final long serialVersionUID = 1L; + private final boolean groupAll; public PreclusteredGroupOperatorDescriptor(IOperatorDescriptorRegistry spec, int[] groupFields, IBinaryComparatorFactory[] comparatorFactories, IAggregatorDescriptorFactory aggregatorFactory, RecordDescriptor recordDescriptor) { + this(spec, groupFields, comparatorFactories, aggregatorFactory, recordDescriptor, false); + } + + public PreclusteredGroupOperatorDescriptor(IOperatorDescriptorRegistry spec, int[] groupFields, + IBinaryComparatorFactory[] comparatorFactories, IAggregatorDescriptorFactory aggregatorFactory, + RecordDescriptor recordDescriptor, boolean groupAll) { super(spec, 1, 1); this.groupFields = groupFields; this.comparatorFactories = comparatorFactories; this.aggregatorFactory = aggregatorFactory; recordDescriptors[0] = recordDescriptor; + this.groupAll = groupAll; } @Override @@ -50,6 +58,6 @@ final IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException { return new PreclusteredGroupOperatorNodePushable(ctx, groupFields, comparatorFactories, aggregatorFactory, - recordDescProvider.getInputRecordDescriptor(getActivityId(), 0), recordDescriptors[0]); + recordDescProvider.getInputRecordDescriptor(getActivityId(), 0), recordDescriptors[0], groupAll); } } diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/preclustered/PreclusteredGroupOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/preclustered/PreclusteredGroupOperatorNodePushable.java index 3286703..2acc4db 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/preclustered/PreclusteredGroupOperatorNodePushable.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/preclustered/PreclusteredGroupOperatorNodePushable.java @@ -35,17 +35,20 @@ private final IAggregatorDescriptorFactory aggregatorFactory; private final RecordDescriptor inRecordDescriptor; private final RecordDescriptor outRecordDescriptor; + private final boolean groupAll; + private PreclusteredGroupWriter pgw; PreclusteredGroupOperatorNodePushable(IHyracksTaskContext ctx, int[] groupFields, IBinaryComparatorFactory[] comparatorFactories, IAggregatorDescriptorFactory aggregatorFactory, - RecordDescriptor inRecordDescriptor, RecordDescriptor outRecordDescriptor) { + RecordDescriptor inRecordDescriptor, RecordDescriptor outRecordDescriptor, boolean groupAll) { this.ctx = ctx; this.groupFields = groupFields; this.comparatorFactories = comparatorFactories; this.aggregatorFactory = aggregatorFactory; this.inRecordDescriptor = inRecordDescriptor; this.outRecordDescriptor = outRecordDescriptor; + this.groupAll = groupAll; } @Override @@ -55,7 +58,7 @@ comparators[i] = comparatorFactories[i].createBinaryComparator(); } pgw = new PreclusteredGroupWriter(ctx, groupFields, comparators, aggregatorFactory, inRecordDescriptor, - outRecordDescriptor, writer); + outRecordDescriptor, writer, false, groupAll); pgw.open(); } diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/preclustered/PreclusteredGroupWriter.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/preclustered/PreclusteredGroupWriter.java index b4e51be..7901141 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/preclustered/PreclusteredGroupWriter.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/preclustered/PreclusteredGroupWriter.java @@ -47,22 +47,28 @@ private final FrameTupleAppenderWrapper appenderWrapper; private final ArrayTupleBuilder tupleBuilder; - private boolean outputPartial = false; - + private final boolean groupAll; + private final boolean outputPartial; private boolean first; - private boolean isFailed = false; public PreclusteredGroupWriter(IHyracksTaskContext ctx, int[] groupFields, IBinaryComparator[] comparators, IAggregatorDescriptorFactory aggregatorFactory, RecordDescriptor inRecordDesc, - RecordDescriptor outRecordDesc, IFrameWriter writer, boolean outputPartial) throws HyracksDataException { - this(ctx, groupFields, comparators, aggregatorFactory, inRecordDesc, outRecordDesc, writer); - this.outputPartial = outputPartial; + RecordDescriptor outRecordDesc, IFrameWriter writer) throws HyracksDataException { + this(ctx, groupFields, comparators, aggregatorFactory, inRecordDesc, outRecordDesc, writer, false, false); } public PreclusteredGroupWriter(IHyracksTaskContext ctx, int[] groupFields, IBinaryComparator[] comparators, IAggregatorDescriptorFactory aggregatorFactory, RecordDescriptor inRecordDesc, - RecordDescriptor outRecordDesc, IFrameWriter writer) throws HyracksDataException { + RecordDescriptor outRecordDesc, IFrameWriter writer, boolean outputPartial) throws HyracksDataException { + this(ctx, groupFields, comparators, aggregatorFactory, inRecordDesc, outRecordDesc, writer, outputPartial, + false); + } + + public PreclusteredGroupWriter(IHyracksTaskContext ctx, int[] groupFields, IBinaryComparator[] comparators, + IAggregatorDescriptorFactory aggregatorFactory, RecordDescriptor inRecordDesc, + RecordDescriptor outRecordDesc, IFrameWriter writer, boolean outputPartial, boolean groupAll) + throws HyracksDataException { this.groupFields = groupFields; this.comparators = comparators; this.aggregator = @@ -79,6 +85,8 @@ appenderWrapper = new FrameTupleAppenderWrapper(appender, writer); tupleBuilder = new ArrayTupleBuilder(outRecordDesc.getFields().length); + this.outputPartial = outputPartial; + this.groupAll = groupAll; } @Override @@ -176,8 +184,7 @@ @Override public void close() throws HyracksDataException { try { - if (!isFailed && !first) { - assert (copyFrameAccessor.getTupleCount() > 0); + if (!isFailed && (!first || groupAll)) { writeOutput(copyFrameAccessor, copyFrameAccessor.getTupleCount() - 1); appenderWrapper.write(); } -- To view, visit https://asterix-gerrit.ics.uci.edu/1185 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I85bb47748950cc909ddbd9720f613e0b8956d320 Gerrit-PatchSet: 6 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Yingyi Bu <buyin...@gmail.com> Gerrit-Reviewer: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Gerrit-Reviewer: Till Westmann <ti...@apache.org> Gerrit-Reviewer: Yingyi Bu <buyin...@gmail.com>