Repository: asterixdb Updated Branches: refs/heads/master ed5030807 -> ef1719e3e
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGroupByVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGroupByVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGroupByVisitor.java index dfe371e..bee5830 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGroupByVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGroupByVisitor.java @@ -18,254 +18,119 @@ */ package org.apache.asterix.lang.sqlpp.rewrites.visitor; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.lang.common.base.Expression; import org.apache.asterix.lang.common.base.ILangExpression; import org.apache.asterix.lang.common.clause.GroupbyClause; -import org.apache.asterix.lang.common.clause.LetClause; -import org.apache.asterix.lang.common.clause.LimitClause; -import org.apache.asterix.lang.common.clause.OrderbyClause; import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair; import org.apache.asterix.lang.common.expression.VariableExpr; import org.apache.asterix.lang.common.rewrites.LangRewritingContext; import org.apache.asterix.lang.common.struct.Identifier; -import org.apache.asterix.lang.common.struct.VarIdentifier; -import org.apache.asterix.lang.sqlpp.clause.FromClause; -import org.apache.asterix.lang.sqlpp.clause.HavingClause; import org.apache.asterix.lang.sqlpp.clause.SelectBlock; import org.apache.asterix.lang.sqlpp.clause.SelectClause; -import org.apache.asterix.lang.sqlpp.expression.SelectExpression; -import org.apache.asterix.lang.sqlpp.util.SqlppRewriteUtil; import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil; -import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppExpressionScopingVisitor; +import org.apache.asterix.lang.sqlpp.visitor.CheckSql92AggregateVisitor; +import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppSimpleExpressionVisitor; import org.apache.hyracks.algebricks.common.utils.Pair; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; + /** - * A pre-processor that adds the group variable as well as its group field - * list into the AST. It will also invoke SQL group-by aggregation sugar rewritings. - */ -// This visitor rewrites non-core SQL++ group-by queries into their SQL++ core version -// queries. For example, for the non-core query in -// asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/sugar-01/sugar-01.3.query.sqlpp, -// -// FROM Employee e -// JOIN Incentive i ON e.job_category = i.job_category -// JOIN SuperStars s ON e.id = s.id -// GROUP BY e.department_id AS deptId -// SELECT deptId as deptId, SUM(e.salary + i.bonus) AS star_cost; -// -// this visitor transforms it into the core version in -// asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/sugar-01/sugar-01.3.query.sqlpp, -// -// FROM Employee e -// JOIN Incentive i ON e.job_category = i.job_category -// JOIN SuperStars s ON e.id = s.id -// GROUP BY e.department_id AS deptId -// GROUP AS eis(e AS e, i AS i, s AS s) -// SELECT ELEMENT { -// 'deptId': deptId, -// 'star_cost': array_sum( (FROM eis AS p SELECT ELEMENT p.e.salary + p.i.bonus) ) -// }; -/** - * The transformation include three things: - * 1. Add a group variable as well as its definition, e.g., GROUP AS eis(e AS e, i AS i, s AS s); - * 2. Rewrite the argument expression of an aggregation function into a subquery if the argument - * expression is not a subquery; - * 3. Turn a SQL-92 aggregate function into a SQL++ core aggregate function when performing 2, e.g., - * SUM(e.salary + i.bonus) becomes - * array_sum( (FROM eis AS p SELECT ELEMENT p.e.salary + p.i.bonus) ). + * A pre-processor that + * <ul> + * <li>adds the group variable as well as its group field + * list into the AST. e.g., GROUP AS eis(e AS e, i AS i, s AS s)</li> + * <li>adds group by clause if select block contains SQL-92 agregate function but there's no group by clause</li> + * </ul> */ +public class SqlppGroupByVisitor extends AbstractSqlppSimpleExpressionVisitor { -public class SqlppGroupByVisitor extends AbstractSqlppExpressionScopingVisitor { + private final LangRewritingContext context; public SqlppGroupByVisitor(LangRewritingContext context) { - super(context); + this.context = context; } @Override public Expression visit(SelectBlock selectBlock, ILangExpression arg) throws CompilationException { - // Traverses the select block in the order of "from", "let"s, "where", - // "group by", "let"s, "having" and "select". - FromClause fromClause = selectBlock.getFromClause(); if (selectBlock.hasFromClause()) { - fromClause.accept(this, arg); - } - if (selectBlock.hasLetClauses()) { - List<LetClause> letList = selectBlock.getLetList(); - for (LetClause letClause : letList) { - letClause.accept(this, arg); + if (selectBlock.hasGroupbyClause()) { + rewriteSelectWithGroupBy(selectBlock, arg); + } else { + rewriteSelectWithoutGroupBy(selectBlock); } } - if (selectBlock.hasWhereClause()) { - selectBlock.getWhereClause().accept(this, arg); - } - if (selectBlock.hasGroupbyClause()) { - GroupbyClause groupbyClause = selectBlock.getGroupbyClause(); - groupbyClause.accept(this, fromClause); - Collection<VariableExpr> visibleVarsInCurrentScope = SqlppVariableUtil.getBindingVariables(groupbyClause); - - VariableExpr groupVar = groupbyClause.getGroupVar(); - Set<VariableExpr> groupFieldVars = getGroupFieldVariables(groupbyClause); - - Collection<VariableExpr> freeVariablesInGbyLets = new HashSet<>(); - if (selectBlock.hasLetClausesAfterGroupby()) { - List<LetClause> letListAfterGby = selectBlock.getLetListAfterGroupby(); - for (LetClause letClauseAfterGby : letListAfterGby) { - letClauseAfterGby.accept(this, arg); - // Rewrites each let clause after the group-by. - SqlppRewriteUtil.rewriteExpressionUsingGroupVariable(groupVar, groupFieldVars, letClauseAfterGby, - context); - Collection<VariableExpr> freeVariablesInLet = SqlppVariableUtil.getFreeVariables(letClauseAfterGby - .getBindingExpr()); - freeVariablesInLet.removeAll(visibleVarsInCurrentScope); - freeVariablesInGbyLets.addAll(freeVariablesInLet); - visibleVarsInCurrentScope.add(letClauseAfterGby.getVarExpr()); - } - } - - Collection<VariableExpr> freeVariables = new HashSet<>(); - if (selectBlock.hasHavingClause()) { - // Rewrites the having clause. - HavingClause havingClause = selectBlock.getHavingClause(); - havingClause.accept(this, arg); - SqlppRewriteUtil.rewriteExpressionUsingGroupVariable(groupVar, groupFieldVars, havingClause, context); - freeVariables.addAll(SqlppVariableUtil.getFreeVariables(havingClause)); - } - - SelectExpression parentSelectExpression = (SelectExpression) arg; - // We cannot rewrite ORDER BY and LIMIT if it's a SET operation query. - if (!parentSelectExpression.getSelectSetOperation().hasRightInputs()) { - if (parentSelectExpression.hasOrderby()) { - // Rewrites the ORDER BY clause. - OrderbyClause orderbyClause = parentSelectExpression.getOrderbyClause(); - orderbyClause.accept(this, arg); - SqlppRewriteUtil.rewriteExpressionUsingGroupVariable(groupVar, groupFieldVars, orderbyClause, - context); - freeVariables.addAll(SqlppVariableUtil.getFreeVariables(orderbyClause)); - } - if (parentSelectExpression.hasLimit()) { - // Rewrites the LIMIT clause. - LimitClause limitClause = parentSelectExpression.getLimitClause(); - limitClause.accept(this, arg); - SqlppRewriteUtil - .rewriteExpressionUsingGroupVariable(groupVar, groupFieldVars, limitClause, context); - freeVariables.addAll(SqlppVariableUtil.getFreeVariables(limitClause)); - } - } - - // Visits the select clause. - SelectClause selectClause = selectBlock.getSelectClause(); - selectClause.accept(this, arg); - // Rewrites the select clause. - SqlppRewriteUtil.rewriteExpressionUsingGroupVariable(groupVar, groupFieldVars, selectClause, context); - freeVariables.addAll(SqlppVariableUtil.getFreeVariables(selectClause)); - freeVariables.removeAll(visibleVarsInCurrentScope); - - // Gets the final free variables. - freeVariables.addAll(freeVariablesInGbyLets); - - // Gets outer scope variables. - Collection<VariableExpr> decorVars = SqlppVariableUtil.getLiveVariables( - scopeChecker.getCurrentScope(), true); - decorVars.removeAll(visibleVarsInCurrentScope); + return super.visit(selectBlock, arg); + } - // Need path resolution or not? - boolean needResolution = !decorVars.containsAll(freeVariables); - // If path resolution is needed, we need to include all outer scope variables in the decoration list. - // Otherwise, we only need to retain used free variables. - if (needResolution) { - // Tracks used variables, including WITH variables. - decorVars.retainAll(freeVariables); - // Adds all outer scope variables, for path resolution. - Collection<VariableExpr> visibleOuterScopeNonWithVars = SqlppVariableUtil.getLiveVariables( - scopeChecker.getCurrentScope(), false); - visibleOuterScopeNonWithVars.removeAll(visibleVarsInCurrentScope); - decorVars.addAll(visibleOuterScopeNonWithVars); - } else { - // Only retains used free variables. - decorVars.retainAll(freeVariables); - } - if (!decorVars.isEmpty()) { - // Adds used WITH variables. - Collection<VariableExpr> visibleOuterScopeNonWithVars = SqlppVariableUtil.getLiveVariables( - scopeChecker.getCurrentScope(), false); - visibleOuterScopeNonWithVars.retainAll(freeVariables); - decorVars.addAll(visibleOuterScopeNonWithVars); + private void rewriteSelectWithGroupBy(SelectBlock selectBlock, ILangExpression arg) throws CompilationException { + GroupbyClause gbyClause = selectBlock.getGroupbyClause(); - // Adds necessary decoration variables for the GROUP BY. - // NOTE: we need to include WITH binding variables so as they can be evaluated before - // the GROUP BY instead of being inlined as part of nested pipepline. The current optimzier - // is not able to optimize the latter case. The following query is such an example: - // asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dapd/q2-11 - List<GbyVariableExpressionPair> decorList = new ArrayList<>(); - for (VariableExpr var : decorVars) { - decorList.add(new GbyVariableExpressionPair((VariableExpr) SqlppRewriteUtil.deepCopy(var), - (Expression) SqlppRewriteUtil.deepCopy(var))); - } - groupbyClause.getDecorPairList().addAll(decorList); - } - } else { - selectBlock.getSelectClause().accept(this, arg); + // Sets the group variable. + if (!gbyClause.hasGroupVar()) { + VariableExpr groupVar = new VariableExpr(context.newVariable()); + gbyClause.setGroupVar(groupVar); } - return null; - } - @Override - public Expression visit(GroupbyClause gc, ILangExpression arg) throws CompilationException { - FromClause fromClause = (FromClause) arg; - Collection<VariableExpr> fromBindingVars = - fromClause == null ? new ArrayList<>() : SqlppVariableUtil.getBindingVariables(fromClause); // Sets the field list for the group variable. - List<Pair<Expression, Identifier>> groupFieldList = new ArrayList<>(); - if (!gc.hasGroupFieldList()) { - for (VariableExpr varExpr : fromBindingVars) { - Pair<Expression, Identifier> varIdPair = new Pair<>(new VariableExpr(varExpr.getVar()), - SqlppVariableUtil.toUserDefinedVariableName(varExpr.getVar())); - groupFieldList.add(varIdPair); - } - gc.setGroupFieldList(groupFieldList); - } else { - for (Pair<Expression, Identifier> groupField : gc.getGroupFieldList()) { + List<Pair<Expression, Identifier>> groupFieldList; + if (gbyClause.hasGroupFieldList()) { + groupFieldList = new ArrayList<>(); + for (Pair<Expression, Identifier> groupField : gbyClause.getGroupFieldList()) { Expression newFieldExpr = groupField.first.accept(this, arg); groupFieldList.add(new Pair<>(newFieldExpr, groupField.second)); } + } else { + groupFieldList = createGroupFieldList(selectBlock); } + gbyClause.setGroupFieldList(groupFieldList); + } - gc.setGroupFieldList(groupFieldList); - - // Sets the group variable. - if (!gc.hasGroupVar()) { + private void rewriteSelectWithoutGroupBy(SelectBlock selectBlock) throws CompilationException { + if (hasSql92Aggregate(selectBlock)) { + // Adds an implicit group-by clause for SQL-92 global aggregate. + List<GbyVariableExpressionPair> gbyPairList = new ArrayList<>(); + List<GbyVariableExpressionPair> decorPairList = new ArrayList<>(); VariableExpr groupVar = new VariableExpr(context.newVariable()); - gc.setGroupVar(groupVar); + List<Pair<Expression, Identifier>> groupFieldList = createGroupFieldList(selectBlock); + GroupbyClause gbyClause = new GroupbyClause(gbyPairList, decorPairList, new HashMap<>(), groupVar, + groupFieldList, false, true); + selectBlock.setGroupbyClause(gbyClause); + } + } + + private boolean hasSql92Aggregate(SelectBlock selectBlock) throws CompilationException { + SelectClause selectClause = selectBlock.getSelectClause(); + if (selectClause.selectRegular()) { + return isSql92Aggregate(selectClause.getSelectRegular(), selectBlock); + } else if (selectClause.selectElement()) { + return isSql92Aggregate(selectClause.getSelectElement(), selectBlock); + } else { + throw new IllegalStateException(); } + } - // Adds the group variable into the "with" (i.e., re-binding) variable list. - VariableExpr gbyVarRef = new VariableExpr(gc.getGroupVar().getVar()); - gbyVarRef.setIsNewVar(false); - Map<Expression, VariableExpr> withVarMap = new HashMap<>(); - withVarMap.put(gbyVarRef, (VariableExpr) SqlppRewriteUtil.deepCopy(gbyVarRef)); - gc.setWithVarMap(withVarMap); + private boolean isSql92Aggregate(ILangExpression expr, SelectBlock selectBlock) throws CompilationException { + CheckSql92AggregateVisitor visitor = new CheckSql92AggregateVisitor(); + return expr.accept(visitor, selectBlock); + } - // Call super.visit(...) to scope variables. - return super.visit(gc, arg); + private List<Pair<Expression, Identifier>> createGroupFieldList(SelectBlock selectBlock) { + List<Pair<Expression, Identifier>> groupFieldList = new ArrayList<>(); + addToGroupFieldList(groupFieldList, SqlppVariableUtil.getBindingVariables(selectBlock.getFromClause())); + addToGroupFieldList(groupFieldList, SqlppVariableUtil.getBindingVariables(selectBlock.getLetList())); + return groupFieldList; } - private Set<VariableExpr> getGroupFieldVariables(GroupbyClause groupbyClause) { - Set<VariableExpr> fieldVars = new HashSet<>(); - if (groupbyClause.hasGroupFieldList()) { - for (Pair<Expression, Identifier> groupField : groupbyClause.getGroupFieldList()) { - fieldVars.add(new VariableExpr(new VarIdentifier(SqlppVariableUtil - .toInternalVariableName(groupField.second.getValue())))); - } + private void addToGroupFieldList(List<Pair<Expression, Identifier>> groupFieldList, + Collection<VariableExpr> fromBindingVars) { + for (VariableExpr varExpr : fromBindingVars) { + Pair<Expression, Identifier> varIdPair = new Pair<>(new VariableExpr(varExpr.getVar()), + SqlppVariableUtil.toUserDefinedVariableName(varExpr.getVar())); + groupFieldList.add(varIdPair); } - return fieldVars; } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java index 6e6ffeb..7a37f94 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java @@ -35,32 +35,27 @@ import org.apache.asterix.lang.common.literal.StringLiteral; import org.apache.asterix.lang.common.rewrites.LangRewritingContext; import org.apache.asterix.lang.common.struct.Identifier; import org.apache.asterix.lang.common.struct.VarIdentifier; +import org.apache.asterix.lang.sqlpp.util.FunctionMapUtil; import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil; import org.apache.asterix.lang.sqlpp.visitor.CheckDatasetOnlyResolutionVisitor; import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppExpressionScopingVisitor; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.om.functions.BuiltinFunctions; +import org.apache.commons.lang3.StringUtils; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; public class VariableCheckAndRewriteVisitor extends AbstractSqlppExpressionScopingVisitor { private static final FunctionSignature FN_DATASET = new FunctionSignature(BuiltinFunctions.DATASET); - protected final boolean overwrite; protected final MetadataProvider metadataProvider; /** - * @param context, - * manages ids of variables and guarantees uniqueness of variables. - * @param overwrite, - * whether rewrite unbounded variables to dataset function calls. - * This flag can only be true for rewriting a top-level query. - * It should be false for rewriting the body expression of a user-defined function. + * @param context, manages ids of variables and guarantees uniqueness of variables. */ - public VariableCheckAndRewriteVisitor(LangRewritingContext context, boolean overwrite, - MetadataProvider metadataProvider) { - super(context); - this.overwrite = overwrite; + public VariableCheckAndRewriteVisitor(LangRewritingContext context, MetadataProvider metadataProvider, + List<VarIdentifier> externalVars) { + super(context, externalVars); this.metadataProvider = metadataProvider; } @@ -74,7 +69,7 @@ public class VariableCheckAndRewriteVisitor extends AbstractSqlppExpressionScopi VariableExpr varExpr = (VariableExpr) leadingExpr; String lastIdentifier = fa.getIdent().getValue(); Expression resolvedExpr = resolve(varExpr, - /** Resolves within the dataverse that has the same name as the variable name. */ + /* Resolves within the dataverse that has the same name as the variable name. */ SqlppVariableUtil.toUserDefinedVariableName(varExpr.getVar().getValue()).getValue(), lastIdentifier, fa, parent); if (resolvedExpr.getKind() == Kind.CALL_EXPRESSION) { @@ -91,58 +86,83 @@ public class VariableCheckAndRewriteVisitor extends AbstractSqlppExpressionScopi @Override public Expression visit(VariableExpr varExpr, ILangExpression parent) throws CompilationException { - return resolve(varExpr, null /** Resolves within the default dataverse. */ - , SqlppVariableUtil.toUserDefinedVariableName(varExpr.getVar().getValue()).getValue(), varExpr, parent); + return resolve(varExpr, null /* Resolves within the default dataverse. */, + SqlppVariableUtil.toUserDefinedVariableName(varExpr.getVar().getValue()).getValue(), varExpr, parent); } // Resolve a variable expression with dataverse name and dataset name. private Expression resolve(VariableExpr varExpr, String dataverseName, String datasetName, - Expression originalExprWithUndefinedIdentifier, ILangExpression parent) - throws CompilationException { + Expression originalExprWithUndefinedIdentifier, ILangExpression parent) throws CompilationException { + String varName = varExpr.getVar().getValue(); - checkError(varName); - if (!rewriteNeeded(varExpr)) { + + VarIdentifier var = lookupVariable(varName); + if (var != null) { + // Exists such an identifier + varExpr.setIsNewVar(false); + varExpr.setVar(var); return varExpr; } - // Note: WITH variables are not used for path resolution. The reason is that - // the accurate typing for ordered list with an UNION item type is not implemented. - // We currently type it as [ANY]. If we include WITH variables for path resolution, - // it will lead to ambiguities and the plan is going to be very complex. An example query is: - // asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/exists - Set<VariableExpr> liveVars = SqlppVariableUtil.getLiveVariables(scopeChecker.getCurrentScope(), false); - boolean resolveToDatasetOnly = resolveToDatasetOnly(originalExprWithUndefinedIdentifier, parent); - boolean resolveAsDataset = datasetExists(dataverseName, datasetName); + boolean resolveToDatasetOnly = resolveToDatasetOnly(originalExprWithUndefinedIdentifier, parent); if (resolveToDatasetOnly) { - if (resolveAsDataset) { - return wrapWithDatasetFunction(dataverseName, datasetName); - } else { - throwUnresolvableError(dataverseName, datasetName); - } + return resolveAsDataset(dataverseName, datasetName); } - return wrapWithResolveFunction(varExpr, liveVars); - } - private void throwUnresolvableError(String dataverseName, String datasetName) throws CompilationException { - String defaultDataverseName = metadataProvider.getDefaultDataverseName(); - if (dataverseName == null && defaultDataverseName == null) { - throw new CompilationException("Cannot find dataset " + datasetName - + " because there is no dataverse declared, nor an alias with name " + datasetName + "!"); + Set<VariableExpr> localVars = scopeChecker.getCurrentScope().getLiveVariables(scopeChecker.getPrecedingScope()); + switch (localVars.size()) { + case 0: + return resolveAsDataset(dataverseName, datasetName); + case 1: + return resolveAsFieldAccess(localVars.iterator().next(), + SqlppVariableUtil.toUserDefinedVariableName(varName).getValue()); + default: + // More than one possibilities. + throw new CompilationException( + "Cannot resolve ambiguous alias reference for undefined identifier " + SqlppVariableUtil + .toUserDefinedVariableName(varName).getValue() + " in " + localVars); } - //If no available dataset nor in-scope variable to resolve to, we throw an error. - throw new CompilationException("Cannot find dataset " + datasetName + " in dataverse " - + (dataverseName == null ? defaultDataverseName : dataverseName) + " nor an alias with name " - + datasetName + "!"); } - // Checks whether we need to error the variable reference, e.g., the variable is referred - // in a LIMIT clause. - private void checkError(String varName) throws CompilationException { + private VarIdentifier lookupVariable(String varName) throws CompilationException { if (scopeChecker.isInForbiddenScopes(varName)) { throw new CompilationException( "Inside limit clauses, it is disallowed to reference a variable having the same name" + " as any variable bound in the same scope as the limit clause."); } + Identifier ident = scopeChecker.lookupSymbol(varName); + return ident != null ? (VarIdentifier) ident : null; + } + + private Expression resolveAsDataset(String dataverseName, String datasetName) throws CompilationException { + if (!datasetExists(dataverseName, datasetName)) { + throwUnresolvableError(dataverseName, datasetName); + } + String fullyQualifiedName = dataverseName == null ? datasetName : dataverseName + "." + datasetName; + List<Expression> argList = new ArrayList<>(1); + argList.add(new LiteralExpr(new StringLiteral(fullyQualifiedName))); + return new CallExpr(new FunctionSignature(BuiltinFunctions.DATASET), argList); + } + + // Rewrites for an field access by name + private Expression resolveAsFieldAccess(VariableExpr var, String fieldName) throws CompilationException { + List<Expression> argList = new ArrayList<>(2); + argList.add(var); + argList.add(new LiteralExpr(new StringLiteral(fieldName))); + return new CallExpr(new FunctionSignature(BuiltinFunctions.FIELD_ACCESS_BY_NAME), argList); + } + + private void throwUnresolvableError(String dataverseName, String datasetName) throws CompilationException { + String defaultDataverseName = metadataProvider.getDefaultDataverseName(); + if (dataverseName == null && defaultDataverseName == null) { + throw new CompilationException("Cannot find dataset " + datasetName + + " because there is no dataverse declared, nor an alias with name " + datasetName + "!"); + } + //If no available dataset nor in-scope variable to resolve to, we throw an error. + throw new CompilationException( + "Cannot find dataset " + datasetName + " in dataverse " + (dataverseName == null ? + defaultDataverseName : + dataverseName) + " nor an alias with name " + datasetName + "!"); } // For a From/Join/UNNEST/Quantifiers binding expression, we resolve the undefined identifier reference as @@ -153,51 +173,29 @@ public class VariableCheckAndRewriteVisitor extends AbstractSqlppExpressionScopi return parent.accept(visitor, originalExpressionWithUndefinedIdentifier); } - // Whether a rewrite is needed for a variable reference expression. - private boolean rewriteNeeded(VariableExpr varExpr) throws CompilationException { - String varName = varExpr.getVar().getValue(); - Identifier ident = scopeChecker.lookupSymbol(varName); - if (ident != null) { - // Exists such an identifier - varExpr.setIsNewVar(false); - varExpr.setVar((VarIdentifier) ident); - return false; - } else { - // Meets a undefined variable - return overwrite; - } - } - - private Expression wrapWithDatasetFunction(String dataverseName, String datasetName) throws CompilationException { - String fullyQualifiedName = dataverseName == null ? datasetName : dataverseName + "." + datasetName; - List<Expression> argList = new ArrayList<>(); - argList.add(new LiteralExpr(new StringLiteral(fullyQualifiedName))); - return new CallExpr(new FunctionSignature(BuiltinFunctions.DATASET), argList); - } - private boolean datasetExists(String dataverseName, String datasetName) throws CompilationException { try { - if (metadataProvider.findDataset(dataverseName, datasetName) != null) { - return true; - } - return fullyQualifiedDatasetNameExists(datasetName); + return metadataProvider.findDataset(dataverseName, datasetName) != null || fullyQualifiedDatasetNameExists( + datasetName); } catch (AlgebricksException e) { throw new CompilationException(e); } } private boolean fullyQualifiedDatasetNameExists(String name) throws AlgebricksException { - if (!name.contains(".")) { + if (name.indexOf('.') < 0) { return false; } - String[] path = name.split("\\."); - if (path.length != 2) { - return false; - } - if (metadataProvider.findDataset(path[0], path[1]) != null) { - return true; - } - return false; + String[] path = StringUtils.split(name, '.'); + return path.length == 2 && metadataProvider.findDataset(path[0], path[1]) != null; } + @Override + public Expression visit(CallExpr callExpr, ILangExpression arg) throws CompilationException { + // skip variables inside SQL-92 aggregates (they will be resolved by SqlppGroupByAggregationSugarVisitor) + if (FunctionMapUtil.isSql92AggregateFunction(callExpr.getFunctionSignature())) { + return callExpr; + } + return super.visit(callExpr, arg); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppRewriteUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppRewriteUtil.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppRewriteUtil.java index a8c5ddc..228ea23 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppRewriteUtil.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppRewriteUtil.java @@ -18,7 +18,6 @@ */ package org.apache.asterix.lang.sqlpp.util; -import java.util.Collection; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -29,7 +28,6 @@ import org.apache.asterix.lang.common.base.ILangExpression; import org.apache.asterix.lang.common.expression.VariableExpr; import org.apache.asterix.lang.common.rewrites.LangRewritingContext; import org.apache.asterix.lang.common.statement.Query; -import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppGroupBySugarVisitor; import org.apache.asterix.lang.sqlpp.visitor.CheckSubqueryVisitor; import org.apache.asterix.lang.sqlpp.visitor.DeepCopyVisitor; import org.apache.asterix.lang.sqlpp.visitor.FreeVariableVisitor; @@ -40,14 +38,6 @@ public class SqlppRewriteUtil { private SqlppRewriteUtil() { } - // Applying sugar rewriting for group-by. - public static Expression rewriteExpressionUsingGroupVariable(VariableExpr groupVar, - Collection<VariableExpr> fieldVars, ILangExpression expr, LangRewritingContext context) - throws CompilationException { - SqlppGroupBySugarVisitor visitor = new SqlppGroupBySugarVisitor(context, groupVar, fieldVars); - return expr.accept(visitor, null); - } - public static Set<VariableExpr> getFreeVariable(Expression expr) throws CompilationException { Set<VariableExpr> vars = new HashSet<>(); FreeVariableVisitor visitor = new FreeVariableVisitor(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java index 2d6ac53..519627c 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java @@ -22,15 +22,12 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.Iterator; import java.util.List; -import java.util.Set; import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.lang.common.base.ILangExpression; import org.apache.asterix.lang.common.clause.GroupbyClause; import org.apache.asterix.lang.common.clause.LetClause; -import org.apache.asterix.lang.common.context.Scope; import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair; import org.apache.asterix.lang.common.expression.VariableExpr; import org.apache.asterix.lang.common.struct.VarIdentifier; @@ -77,22 +74,6 @@ public class SqlppVariableUtil { return varName; } - public static Set<VariableExpr> getLiveVariables(Scope scope, boolean includeWithVariables) { - Set<VariableExpr> results = new HashSet<>(); - Set<VariableExpr> liveVars = scope.getLiveVariables(); - Iterator<VariableExpr> liveVarIter = liveVars.iterator(); - while (liveVarIter.hasNext()) { - VariableExpr liveVar = liveVarIter.next(); - // Variables defined in WITH clauses are named value access. - // TODO(buyingi): remove this if block once we can accurately type - // ordered lists with UNION item type. Currently it is typed as [ANY]. - if (includeWithVariables || !liveVar.getVar().namedValueAccess()) { - results.add(liveVar); - } - } - return results; - } - public static String toInternalVariableName(String varName) { return USER_VAR_PREFIX + varName; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/DeepCopyVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/DeepCopyVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/DeepCopyVisitor.java index 0d3f320..ffd63d4 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/DeepCopyVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/DeepCopyVisitor.java @@ -231,7 +231,8 @@ public class DeepCopyVisitor extends AbstractSqlppQueryExpressionVisitor<ILangEx @Override public Query visit(Query q, Void arg) throws CompilationException { - return new Query(q.isExplain(), q.isTopLevel(), (Expression) q.getBody().accept(this, arg), q.getVarCounter()); + return new Query(q.isExplain(), q.isTopLevel(), (Expression) q.getBody().accept(this, arg), q.getVarCounter(), + q.getExternalVars()); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java index 6c38334..1a54b92 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java @@ -166,7 +166,8 @@ public class FreeVariableVisitor extends AbstractSqlppQueryExpressionVisitor<Voi selectBlock.getSelectClause().accept(this, selectFreeVars); // Removes group-by, from, let, and gby-let binding vars. - removeAllBindingVarsInSelectBlock(selectFreeVars, fromBindingVars, letsBindingVars, gbyLetsBindingVars); + removeAllBindingVarsInSelectBlock(selectFreeVars, fromBindingVars, letsBindingVars, gbyBindingVars, + gbyLetsBindingVars); if (selectBlock.hasFromClause()) { selectBlock.getFromClause().accept(this, fromFreeVars); @@ -193,13 +194,15 @@ public class FreeVariableVisitor extends AbstractSqlppQueryExpressionVisitor<Voi } if (selectBlock.hasHavingClause()) { selectBlock.getHavingClause().accept(this, selectFreeVars); - removeAllBindingVarsInSelectBlock(selectFreeVars, fromBindingVars, letsBindingVars, gbyLetsBindingVars); + removeAllBindingVarsInSelectBlock(selectFreeVars, fromBindingVars, letsBindingVars, gbyBindingVars, + gbyLetsBindingVars); } } // Removes all binding vars from <code>freeVars</code>, which contains the free // vars in the order-by and limit. - removeAllBindingVarsInSelectBlock(freeVars, fromBindingVars, letsBindingVars, gbyLetsBindingVars); + removeAllBindingVarsInSelectBlock(freeVars, fromBindingVars, letsBindingVars, gbyBindingVars, + gbyLetsBindingVars); // Adds all free vars. freeVars.addAll(selectFreeVars); @@ -462,23 +465,23 @@ public class FreeVariableVisitor extends AbstractSqlppQueryExpressionVisitor<Voi /** * Removes all binding variables defined in the select block for a free variable collection. - * - * @param freeVars, + * @param selectFreeVars, * free variables. * @param fromBindingVars, * binding variables defined in the from clause of a select block. * @param letsBindingVars, * binding variables defined in the let clauses of the select block. - * @param gbyLetsBindingVars, - * binding variables defined in the let clauses after a group-by in the select block. + * @param gbyBindingVars + * binding variables defined in the groupby clauses of the select block + * @param gbyLetsBindingVars + * binding variables defined in the let clauses after groupby of the select block. */ private void removeAllBindingVarsInSelectBlock(Collection<VariableExpr> selectFreeVars, Collection<VariableExpr> fromBindingVars, Collection<VariableExpr> letsBindingVars, - Collection<VariableExpr> gbyLetsBindingVars) { + Collection<VariableExpr> gbyBindingVars, Collection<VariableExpr> gbyLetsBindingVars) { selectFreeVars.removeAll(fromBindingVars); selectFreeVars.removeAll(letsBindingVars); - selectFreeVars.removeAll(gbyLetsBindingVars); + selectFreeVars.removeAll(gbyBindingVars); selectFreeVars.removeAll(gbyLetsBindingVars); } - } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppAstPrintVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppAstPrintVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppAstPrintVisitor.java index 38d09f9..09eaa59 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppAstPrintVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppAstPrintVisitor.java @@ -20,6 +20,7 @@ package org.apache.asterix.lang.sqlpp.visitor; import java.io.PrintWriter; import java.util.List; +import java.util.Map; import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.functions.FunctionSignature; @@ -28,6 +29,7 @@ import org.apache.asterix.lang.common.clause.GroupbyClause; import org.apache.asterix.lang.common.clause.LetClause; import org.apache.asterix.lang.common.expression.CallExpr; import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair; +import org.apache.asterix.lang.common.expression.VariableExpr; import org.apache.asterix.lang.common.struct.Identifier; import org.apache.asterix.lang.common.visitor.QueryPrintVisitor; import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause; @@ -264,15 +266,25 @@ public class SqlppAstPrintVisitor extends QueryPrintVisitor implements ISqlppVis public Void visit(GroupbyClause gc, Integer step) throws CompilationException { if (gc.isGroupAll()) { out.println(skip(step) + "Group All"); - return null; + } else { + out.println(skip(step) + "Groupby"); + for (GbyVariableExpressionPair pair : gc.getGbyPairList()) { + if (pair.getVar() != null) { + pair.getVar().accept(this, step + 1); + out.println(skip(step + 1) + ":="); + } + pair.getExpr().accept(this, step + 1); + } } - out.println(skip(step) + "Groupby"); - for (GbyVariableExpressionPair pair : gc.getGbyPairList()) { - if (pair.getVar() != null) { - pair.getVar().accept(this, step + 1); - out.println(skip(step + 1) + ":="); + if (gc.hasDecorList()) { + out.println(skip(step + 1) + "DECOR"); + for (GbyVariableExpressionPair pair : gc.getDecorPairList()) { + if (pair.getVar() != null) { + pair.getVar().accept(this, step + 1); + out.println(skip(step + 1) + ":="); + } + pair.getExpr().accept(this, step + 1); } - pair.getExpr().accept(this, step + 1); } if (gc.hasGroupVar()) { out.print(skip(step + 1) + "GROUP AS "); @@ -286,6 +298,14 @@ public class SqlppAstPrintVisitor extends QueryPrintVisitor implements ISqlppVis out.println(skip(step + 1) + ")"); } } + if (gc.hasWithMap()) { + out.println(skip(step + 1) + "WITH"); + for (Map.Entry<Expression, VariableExpr> pair : gc.getWithVarMap().entrySet()) { + pair.getValue().accept(this, step + 1); + out.println(skip(step + 1) + ":="); + pair.getKey().accept(this, step + 1); + } + } out.println(); return null; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppCloneAndSubstituteVariablesVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppCloneAndSubstituteVariablesVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppCloneAndSubstituteVariablesVisitor.java index 844856d..65e9255 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppCloneAndSubstituteVariablesVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppCloneAndSubstituteVariablesVisitor.java @@ -223,7 +223,7 @@ public class SqlppCloneAndSubstituteVariablesVisitor extends CloneAndSubstituteV for (LetClause letClause : selectBlock.getLetList()) { newLet = letClause.accept(this, currentEnv); currentEnv = newLet.second; - newLetClauses.add(letClause); + newLetClauses.add((LetClause) newLet.first); } } @@ -346,7 +346,7 @@ public class SqlppCloneAndSubstituteVariablesVisitor extends CloneAndSubstituteV if (selectExpression.hasLetClauses()) { for (LetClause letClause : selectExpression.getLetList()) { p = letClause.accept(this, currentEnv); - newLetList.add(letClause); + newLetList.add((LetClause) p.first); currentEnv = p.second; } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppDeleteRewriteVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppDeleteRewriteVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppDeleteRewriteVisitor.java index 4cda253..4542bab 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppDeleteRewriteVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppDeleteRewriteVisitor.java @@ -22,7 +22,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import org.apache.asterix.common.functions.FunctionConstants; import org.apache.asterix.common.functions.FunctionSignature; import org.apache.asterix.lang.common.base.Expression; import org.apache.asterix.lang.common.clause.WhereClause; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java index 2f9cb32..737cc53 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java @@ -18,7 +18,6 @@ */ package org.apache.asterix.lang.sqlpp.visitor.base; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -26,19 +25,15 @@ import java.util.Map.Entry; import java.util.Set; import org.apache.asterix.common.exceptions.CompilationException; -import org.apache.asterix.common.functions.FunctionSignature; import org.apache.asterix.lang.common.base.Expression; import org.apache.asterix.lang.common.base.ILangExpression; import org.apache.asterix.lang.common.clause.GroupbyClause; import org.apache.asterix.lang.common.clause.LetClause; import org.apache.asterix.lang.common.clause.LimitClause; import org.apache.asterix.lang.common.context.Scope; -import org.apache.asterix.lang.common.expression.CallExpr; import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair; -import org.apache.asterix.lang.common.expression.LiteralExpr; import org.apache.asterix.lang.common.expression.QuantifiedExpression; import org.apache.asterix.lang.common.expression.VariableExpr; -import org.apache.asterix.lang.common.literal.StringLiteral; import org.apache.asterix.lang.common.parser.ScopeChecker; import org.apache.asterix.lang.common.rewrites.LangRewritingContext; import org.apache.asterix.lang.common.statement.FunctionDecl; @@ -57,7 +52,6 @@ import org.apache.asterix.lang.sqlpp.clause.UnnestClause; import org.apache.asterix.lang.sqlpp.expression.SelectExpression; import org.apache.asterix.lang.sqlpp.struct.SetOperationRight; import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil; -import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.algebricks.core.algebra.base.Counter; @@ -71,8 +65,23 @@ public class AbstractSqlppExpressionScopingVisitor extends AbstractSqlppSimpleEx * manages ids of variables and guarantees uniqueness of variables. */ public AbstractSqlppExpressionScopingVisitor(LangRewritingContext context) { + this(context, null); + } + + /** + * @param context, + * manages ids of variables and guarantees uniqueness of variables. + * @param externalVars + * pre-defined (external) variables that must be added to the initial scope + */ + public AbstractSqlppExpressionScopingVisitor(LangRewritingContext context, List<VarIdentifier> externalVars) { this.context = context; this.scopeChecker.setVarCounter(new Counter(context.getVarCounter())); + if (externalVars != null) { + for (VarIdentifier paramVar : externalVars) { + scopeChecker.getCurrentScope().addSymbolToScope(paramVar); + } + } } @Override @@ -219,7 +228,7 @@ public class AbstractSqlppExpressionScopingVisitor extends AbstractSqlppSimpleEx public Expression visit(GroupbyClause gc, ILangExpression arg) throws CompilationException { // After a GROUP BY, variables defined before the current SELECT BLOCK (e.g., in a WITH clause // or an outer scope query) should still be visible. - Scope newScope = new Scope(scopeChecker, scopeChecker.getCurrentScope().getParentScope()); + Scope newScope = new Scope(scopeChecker, scopeChecker.getPrecedingScope()); // Puts all group-by variables into the symbol set of the new scope. for (GbyVariableExpressionPair gbyKeyVarExpr : gc.getGbyPairList()) { gbyKeyVarExpr.setExpr(visit(gbyKeyVarExpr.getExpr(), gc)); @@ -243,7 +252,7 @@ public class AbstractSqlppExpressionScopingVisitor extends AbstractSqlppSimpleEx } } if (gc.hasGroupVar()) { - addNewVarSymbolToScope(scopeChecker.getCurrentScope(), gc.getGroupVar().getVar()); + addNewVarSymbolToScope(newScope, gc.getGroupVar().getVar()); } if (gc.hasWithMap()) { Map<Expression, VariableExpr> newWithMap = new HashMap<>(); @@ -288,8 +297,6 @@ public class AbstractSqlppExpressionScopingVisitor extends AbstractSqlppSimpleEx // visit let list if (selectExpression.hasLetClauses()) { for (LetClause letClause : selectExpression.getLetList()) { - // Variables defined in WITH clauses are considered as named access instead of real variables. - letClause.getVarExpr().getVar().setNamedValueAccess(true); letClause.accept(this, selectExpression); } scopeChecker.createNewScope(); @@ -365,17 +372,6 @@ public class AbstractSqlppExpressionScopingVisitor extends AbstractSqlppSimpleEx return null; } - // Rewrites for an undefined variable reference, which potentially could be a syntatic sugar. - protected Expression wrapWithResolveFunction(VariableExpr expr, Set<VariableExpr> liveVars) - throws CompilationException { - List<Expression> argList = new ArrayList<>(); - //Ignore the parser-generated prefix "$" for a dataset. - String varName = SqlppVariableUtil.toUserDefinedVariableName(expr.getVar().getValue()).getValue(); - argList.add(new LiteralExpr(new StringLiteral(varName))); - argList.addAll(liveVars); - return new CallExpr(new FunctionSignature(BuiltinFunctions.RESOLVE), argList); - } - // Adds a new encountered alias identifier into a scope private void addNewVarSymbolToScope(Scope scope, VarIdentifier var) throws CompilationException { if (scope.findLocalSymbol(var.getValue()) != null) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj index c5d6d26..2bf84e0 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj +++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj @@ -143,6 +143,7 @@ import org.apache.asterix.lang.common.statement.UpdateStatement; import org.apache.asterix.lang.common.statement.UpsertStatement; import org.apache.asterix.lang.common.statement.WriteStatement; import org.apache.asterix.lang.common.struct.Identifier; +import org.apache.asterix.lang.common.struct.OperatorType; import org.apache.asterix.lang.common.struct.QuantifiedPair; import org.apache.asterix.lang.common.struct.VarIdentifier; import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause; @@ -1973,6 +1974,7 @@ Expression BetweenExpr()throws ParseException: <AND> operand = IsExpr() { + op.addOperator(OperatorType.AND); op.addOperand(operand); } )? http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java index e59c600..e1f5bb0 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java @@ -871,8 +871,6 @@ public class BuiltinFunctions { FunctionIdentifier.VARARGS); public static final FunctionIdentifier META_KEY = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "meta-key", FunctionIdentifier.VARARGS); - public static final FunctionIdentifier RESOLVE = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "resolve", - FunctionIdentifier.VARARGS); public static IFunctionInfo getAsterixFunctionInfo(FunctionIdentifier fid) { return registeredFunctions.get(fid); @@ -1184,7 +1182,7 @@ public class BuiltinFunctions { addPrivateFunction(OPEN_RECORD_CONSTRUCTOR, OpenRecordConstructorResultType.INSTANCE, true); addPrivateFunction(FIELD_ACCESS_BY_INDEX, FieldAccessByIndexResultType.INSTANCE, true); addPrivateFunction(FIELD_ACCESS_NESTED, FieldAccessNestedResultType.INSTANCE, true); - addPrivateFunction(FIELD_ACCESS_BY_NAME, FieldAccessByNameResultType.INSTANCE, true); + addFunction(FIELD_ACCESS_BY_NAME, FieldAccessByNameResultType.INSTANCE, true); addFunction(GET_RECORD_FIELDS, OrderedListOfAnyTypeComputer.INSTANCE, true); addFunction(GET_RECORD_FIELD_VALUE, FieldAccessNestedResultType.INSTANCE, true); addFunction(RECORD_PAIRS, RecordPairsTypeComputer.INSTANCE, true); @@ -1270,7 +1268,6 @@ public class BuiltinFunctions { // meta() function addFunction(META, OpenARecordTypeComputer.INSTANCE, true); addPrivateFunction(META_KEY, AnyTypeComputer.INSTANCE, false); - addFunction(RESOLVE, AnyTypeComputer.INSTANCE, false); addPrivateFunction(COLLECTION_TO_SEQUENCE, CollectionToSequenceTypeComputer.INSTANCE, true); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef1719e3/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java index 9e7daf0..eecd066 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java @@ -179,18 +179,23 @@ public class ExternalGroupByPOperator extends AbstractPhysicalOperator { Mutable<ILogicalOperator> r0 = p0.getRoots().get(0); AggregateOperator aggOp = (AggregateOperator) r0.getValue(); + compileSubplans(inputSchemas[0], gby, opSchema, context); + IPartialAggregationTypeComputer partialAggregationTypeComputer = context.getPartialAggregationTypeComputer(); List<Object> intermediateTypes = new ArrayList<Object>(); int n = aggOp.getExpressions().size(); ISerializedAggregateEvaluatorFactory[] aff = new ISerializedAggregateEvaluatorFactory[n]; int i = 0; IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider(); - IVariableTypeEnvironment aggOpInputEnv = context.getTypeEnvironment(aggOp.getInputs().get(0).getValue()); + ILogicalOperator aggOpInput = aggOp.getInputs().get(0).getValue(); + IOperatorSchema aggOpInputSchema = context.getSchema(aggOpInput); + IOperatorSchema[] aggOpInputSchemas = new IOperatorSchema[] { aggOpInputSchema }; + IVariableTypeEnvironment aggOpInputEnv = context.getTypeEnvironment(aggOpInput); IVariableTypeEnvironment outputEnv = context.getTypeEnvironment(op); for (Mutable<ILogicalExpression> exprRef : aggOp.getExpressions()) { AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) exprRef.getValue(); aff[i++] = expressionRuntimeProvider.createSerializableAggregateFunctionFactory(aggFun, aggOpInputEnv, - inputSchemas, context); + aggOpInputSchemas, context); intermediateTypes .add(partialAggregationTypeComputer.getType(aggFun, aggOpInputEnv, context.getMetadataProvider())); } @@ -215,7 +220,6 @@ public class ExternalGroupByPOperator extends AbstractPhysicalOperator { aggOpInputEnv.setVarType(var, outputEnv.getVarType(var)); } - compileSubplans(inputSchemas[0], gby, opSchema, context); IOperatorDescriptorRegistry spec = builder.getJobSpec(); IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(gbyCols, aggOpInputEnv, context);
