http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppCloneAndSubstituteVariablesVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppCloneAndSubstituteVariablesVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppCloneAndSubstituteVariablesVisitor.java index 0c70572..39e1883 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppCloneAndSubstituteVariablesVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppCloneAndSubstituteVariablesVisitor.java @@ -33,6 +33,7 @@ import org.apache.asterix.lang.common.clause.WhereClause; import org.apache.asterix.lang.common.expression.VariableExpr; import org.apache.asterix.lang.common.rewrites.LangRewritingContext; import org.apache.asterix.lang.common.rewrites.VariableSubstitutionEnvironment; +import org.apache.asterix.lang.common.struct.Identifier; import org.apache.asterix.lang.common.util.VariableCloneAndSubstitutionUtil; import org.apache.asterix.lang.common.visitor.CloneAndSubstituteVariablesVisitor; import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause; @@ -90,7 +91,7 @@ public class SqlppCloneAndSubstituteVariablesVisitor extends CloneAndSubstituteV VariableExpr newLeftVar = generateNewVariable(context, leftVar); VariableExpr newLeftPosVar = fromTerm.hasPositionalVariable() ? generateNewVariable(context, fromTerm.getPositionalVariable()) : null; - Expression newLeftExpr = (Expression) visitUnnesBindingExpression(fromTerm.getLeftExpression(), env).first; + Expression newLeftExpr = (Expression) visitUnnestBindingExpression(fromTerm.getLeftExpression(), env).first; List<AbstractBinaryCorrelateClause> newCorrelateClauses = new ArrayList<>(); VariableSubstitutionEnvironment currentEnv = new VariableSubstitutionEnvironment(env); @@ -131,7 +132,7 @@ public class SqlppCloneAndSubstituteVariablesVisitor extends CloneAndSubstituteV ? generateNewVariable(context, joinClause.getPositionalVariable()) : null; // Visits the right expression. - Expression newRightExpr = (Expression) visitUnnesBindingExpression(joinClause.getRightExpression(), env).first; + Expression newRightExpr = (Expression) visitUnnestBindingExpression(joinClause.getRightExpression(), env).first; // Visits the condition. VariableSubstitutionEnvironment currentEnv = new VariableSubstitutionEnvironment(env); @@ -183,7 +184,7 @@ public class SqlppCloneAndSubstituteVariablesVisitor extends CloneAndSubstituteV ? generateNewVariable(context, unnestClause.getPositionalVariable()) : null; // Visits the right expression. - Expression rightExpr = (Expression) visitUnnesBindingExpression(unnestClause.getRightExpression(), env).first; + Expression rightExpr = (Expression) visitUnnestBindingExpression(unnestClause.getRightExpression(), env).first; // Visits the condition. VariableSubstitutionEnvironment currentEnv = new VariableSubstitutionEnvironment(env); @@ -416,14 +417,26 @@ public class SqlppCloneAndSubstituteVariablesVisitor extends CloneAndSubstituteV @Override public Pair<ILangExpression, VariableSubstitutionEnvironment> visit(WindowExpression winExpr, VariableSubstitutionEnvironment env) throws CompilationException { - Expression newExpr = (Expression) winExpr.getExpr().accept(this, env).first; + List<Expression> newExprList = + VariableCloneAndSubstitutionUtil.visitAndCloneExprList(winExpr.getExprList(), env, this); List<Expression> newPartitionList = winExpr.hasPartitionList() ? VariableCloneAndSubstitutionUtil.visitAndCloneExprList(winExpr.getPartitionList(), env, this) : null; - List<Expression> newOrderbyList = - VariableCloneAndSubstitutionUtil.visitAndCloneExprList(winExpr.getOrderbyList(), env, this); - List<OrderbyClause.OrderModifier> newOrderbyModifierList = new ArrayList<>(winExpr.getOrderbyModifierList()); - WindowExpression newWinExpr = - new WindowExpression(newExpr, newPartitionList, newOrderbyList, newOrderbyModifierList); + List<Expression> newOrderbyList = winExpr.hasOrderByList() + ? VariableCloneAndSubstitutionUtil.visitAndCloneExprList(winExpr.getOrderbyList(), env, this) : null; + List<OrderbyClause.OrderModifier> newOrderbyModifierList = + winExpr.hasOrderByList() ? new ArrayList<>(winExpr.getOrderbyModifierList()) : null; + Expression newFrameStartExpr = + winExpr.hasFrameStartExpr() ? (Expression) winExpr.getFrameStartExpr().accept(this, env).first : null; + Expression newFrameEndExpr = + winExpr.hasFrameEndExpr() ? (Expression) winExpr.getFrameEndExpr().accept(this, env).first : null; + VariableExpr newWindowVar = + winExpr.hasWindowVar() ? (VariableExpr) winExpr.getWindowVar().accept(this, env).first : null; + List<Pair<Expression, Identifier>> newWindowFieldList = winExpr.hasWindowFieldList() + ? VariableCloneAndSubstitutionUtil.substInFieldList(winExpr.getWindowFieldList(), env, this) : null; + WindowExpression newWinExpr = new WindowExpression(winExpr.getFunctionSignature(), newExprList, + newPartitionList, newOrderbyList, newOrderbyModifierList, winExpr.getFrameMode(), + winExpr.getFrameStartKind(), newFrameStartExpr, winExpr.getFrameEndKind(), newFrameEndExpr, + winExpr.getFrameExclusionKind(), newWindowVar, newWindowFieldList); newWinExpr.setSourceLocation(winExpr.getSourceLocation()); newWinExpr.addHints(winExpr.getHints()); return new Pair<>(newWinExpr, env);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppFormatPrintVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppFormatPrintVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppFormatPrintVisitor.java index 99368f8..1ee3004 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppFormatPrintVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppFormatPrintVisitor.java @@ -335,8 +335,16 @@ public class SqlppFormatPrintVisitor extends FormatPrintVisitor implements ISqlp @Override public Void visit(WindowExpression windowExpr, Integer step) throws CompilationException { out.print(skip(step) + "window "); - windowExpr.getExpr().accept(this, step + 2); - out.print(skip(step) + " over ("); + out.print(generateFullName(windowExpr.getFunctionSignature().getNamespace(), + windowExpr.getFunctionSignature().getName()) + "("); + printDelimitedExpressions(windowExpr.getExprList(), COMMA, step); + out.print(")"); + out.print(skip(step) + " over "); + if (windowExpr.hasWindowVar()) { + windowExpr.getWindowVar().accept(this, step + 2); + out.print(skip(step) + "as "); + } + out.print("("); if (windowExpr.hasPartitionList()) { List<Expression> partitionList = windowExpr.getPartitionList(); for (int i = 0, ln = partitionList.size(); i < ln; i++) { @@ -347,8 +355,22 @@ public class SqlppFormatPrintVisitor extends FormatPrintVisitor implements ISqlp partExpr.accept(this, step + 2); } } - out.print(" order by "); - printDelimitedObyExpressions(windowExpr.getOrderbyList(), windowExpr.getOrderbyModifierList(), step + 2); + if (windowExpr.hasOrderByList()) { + out.print(skip(step) + " order by "); + printDelimitedObyExpressions(windowExpr.getOrderbyList(), windowExpr.getOrderbyModifierList(), step + 2); + } + if (windowExpr.hasFrameDefinition()) { + out.println(skip(step) + windowExpr.getFrameMode()); + if (windowExpr.hasFrameStartExpr()) { + windowExpr.getFrameStartExpr().accept(this, step + 2); + } + out.println(skip(step) + windowExpr.getFrameStartKind()); + if (windowExpr.hasFrameEndExpr()) { + windowExpr.getFrameEndExpr().accept(this, step + 2); + } + out.println(skip(step) + windowExpr.getFrameEndKind()); + out.println(skip(step) + "exclude " + windowExpr.getFrameExclusionKind()); + } out.println(skip(step) + ")"); return null; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java index df165c0..8650eec 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java @@ -51,6 +51,7 @@ import org.apache.asterix.lang.sqlpp.clause.NestClause; import org.apache.asterix.lang.sqlpp.clause.SelectSetOperation; import org.apache.asterix.lang.sqlpp.clause.UnnestClause; import org.apache.asterix.lang.sqlpp.expression.SelectExpression; +import org.apache.asterix.lang.sqlpp.expression.WindowExpression; import org.apache.asterix.lang.sqlpp.struct.SetOperationRight; import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil; import org.apache.hyracks.algebricks.common.utils.Pair; @@ -375,6 +376,22 @@ public class AbstractSqlppExpressionScopingVisitor extends AbstractSqlppSimpleEx return null; } + @Override + public Expression visit(WindowExpression winExpr, ILangExpression arg) throws CompilationException { + visitWindowExpressionExcludingExprList(winExpr, arg); + if (winExpr.hasWindowVar()) { + Scope preScope = scopeChecker.getCurrentScope(); + Scope newScope = scopeChecker.extendCurrentScope(); + VariableExpr windowVar = winExpr.getWindowVar(); + addNewVarSymbolToScope(newScope, windowVar.getVar(), windowVar.getSourceLocation()); + winExpr.setExprList(visit(winExpr.getExprList(), arg)); + scopeChecker.replaceCurrentScope(preScope); + } else { + winExpr.setExprList(visit(winExpr.getExprList(), arg)); + } + return winExpr; + } + // Adds a new encountered alias identifier into a scope private void addNewVarSymbolToScope(Scope scope, VarIdentifier var, SourceLocation sourceLoc) throws CompilationException { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java index 9454984..d1092b3 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java @@ -46,6 +46,7 @@ import org.apache.asterix.lang.common.expression.VariableExpr; import org.apache.asterix.lang.common.statement.FunctionDecl; import org.apache.asterix.lang.common.statement.InsertStatement; import org.apache.asterix.lang.common.statement.Query; +import org.apache.asterix.lang.common.struct.Identifier; import org.apache.asterix.lang.common.struct.QuantifiedPair; import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause; import org.apache.asterix.lang.sqlpp.clause.FromClause; @@ -64,6 +65,7 @@ import org.apache.asterix.lang.sqlpp.expression.CaseExpression; import org.apache.asterix.lang.sqlpp.expression.SelectExpression; import org.apache.asterix.lang.sqlpp.expression.WindowExpression; import org.apache.asterix.lang.sqlpp.struct.SetOperationRight; +import org.apache.hyracks.algebricks.common.utils.Pair; public class AbstractSqlppSimpleExpressionVisitor extends AbstractSqlppQueryExpressionVisitor<Expression, ILangExpression> { @@ -217,8 +219,10 @@ public class AbstractSqlppSimpleExpressionVisitor for (GbyVariableExpressionPair gbyVarExpr : gc.getGbyPairList()) { gbyVarExpr.setExpr(visit(gbyVarExpr.getExpr(), gc)); } - for (GbyVariableExpressionPair decVarExpr : gc.getDecorPairList()) { - decVarExpr.setExpr(visit(decVarExpr.getExpr(), gc)); + if (gc.hasDecorList()) { + for (GbyVariableExpressionPair decVarExpr : gc.getDecorPairList()) { + decVarExpr.setExpr(visit(decVarExpr.getExpr(), gc)); + } } return null; } @@ -324,12 +328,30 @@ public class AbstractSqlppSimpleExpressionVisitor @Override public Expression visit(WindowExpression winExpr, ILangExpression arg) throws CompilationException { - winExpr.setExpr(visit(winExpr.getExpr(), arg)); + visitWindowExpressionExcludingExprList(winExpr, arg); + winExpr.setExprList(visit(winExpr.getExprList(), arg)); + return winExpr; + } + + protected void visitWindowExpressionExcludingExprList(WindowExpression winExpr, ILangExpression arg) + throws CompilationException { if (winExpr.hasPartitionList()) { winExpr.setPartitionList(visit(winExpr.getPartitionList(), winExpr)); } - winExpr.setOrderbyList(visit(winExpr.getOrderbyList(), winExpr)); - return winExpr; + if (winExpr.hasOrderByList()) { + winExpr.setOrderbyList(visit(winExpr.getOrderbyList(), winExpr)); + } + if (winExpr.hasFrameStartExpr()) { + winExpr.setFrameStartExpr(visit(winExpr.getFrameStartExpr(), winExpr)); + } + if (winExpr.hasFrameEndExpr()) { + winExpr.setFrameEndExpr(visit(winExpr.getFrameEndExpr(), winExpr)); + } + if (winExpr.hasWindowFieldList()) { + for (Pair<Expression, Identifier> field : winExpr.getWindowFieldList()) { + field.first = visit(field.first, arg); + } + } } @Override @@ -391,8 +413,8 @@ public class AbstractSqlppSimpleExpressionVisitor return expr; } - private List<Expression> visit(List<Expression> exprs, ILangExpression arg) throws CompilationException { - List<Expression> newExprList = new ArrayList<>(); + protected List<Expression> visit(List<Expression> exprs, ILangExpression arg) throws CompilationException { + List<Expression> newExprList = new ArrayList<>(exprs.size()); for (Expression expr : exprs) { newExprList.add(visit(expr, arg)); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj index 19f2cee..4078389 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj +++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj @@ -186,6 +186,21 @@ import org.apache.hyracks.api.exceptions.SourceLocation; class SQLPPParser extends ScopeChecker implements IParser { + // tokens parsed as identifiers + private static final String CURRENT = "CURRENT"; + private static final String EXCLUDE = "EXCLUDE"; + private static final String FOLLOWING = "FOLLOWING"; + private static final String GROUPS = "GROUPS"; + private static final String NO = "NO"; + private static final String OTHERS = "OTHERS"; + private static final String PARTITION = "PARTITION"; + private static final String PRECEDING = "PRECEDING"; + private static final String RANGE = "RANGE"; + private static final String ROW = "ROW"; + private static final String ROWS = "ROWS"; + private static final String TIES = "TIES"; + private static final String UNBOUNDED = "UNBOUNDED"; + // optimizer hints private static final String AUTO_HINT = "auto"; private static final String BROADCAST_JOIN_HINT = "bcast"; @@ -372,6 +387,33 @@ class SQLPPParser extends ScopeChecker implements IParser { expr.setSourceLocation(getSourceLocation(token)); return expr; } + + private boolean isToken(String image) { + return token.image.equalsIgnoreCase(image); + } + + private void expectToken(String image) throws SqlppParseException { + if (!isToken(image)) { + throw createUnexpectedTokenError(); + } + } + + private SqlppParseException createUnexpectedTokenError() { + return new SqlppParseException(getSourceLocation(token), "Unexpected token: " + token.image); + } + + private boolean laToken(int idx, int kind, String image) { + Token t = getToken(idx); + return t.kind == kind && t.image.equalsIgnoreCase(image); + } + + private boolean laIdentifier(int idx, String image) { + return laToken(idx, IDENTIFIER, image); + } + + private boolean laIdentifier(String image) { + return laIdentifier(1, image); + } } PARSER_END(SQLPPParser) @@ -2489,6 +2531,36 @@ VariableExpr Variable() throws ParseException: } } +Pair<VariableExpr, List<Pair<Expression, Identifier>>> VariableWithFieldMap() throws ParseException: +{ + VariableExpr var = null; + List<Pair<Expression, Identifier>> fieldList = new ArrayList<Pair<Expression, Identifier>>(); +} +{ + var = Variable() + ( LOOKAHEAD(1) + { + VariableExpr fieldVarExpr = null; + String fieldIdentifierStr = null; + } + <LEFTPAREN> + fieldVarExpr = VariableRef() <AS> fieldIdentifierStr = Identifier() + { + fieldList.add(new Pair<Expression, Identifier>(fieldVarExpr, new Identifier(fieldIdentifierStr))); + } + (<COMMA> + fieldVarExpr = VariableRef() <AS> fieldIdentifierStr = Identifier() + { + fieldList.add(new Pair<Expression, Identifier>(fieldVarExpr, new Identifier(fieldIdentifierStr))); + } + )* + <RIGHTPAREN> + )? + { + return new Pair<VariableExpr, List<Pair<Expression, Identifier>>>(var, fieldList); + } +} + VariableExpr ExternalVariableRef() throws ParseException: { String name = null; @@ -2615,6 +2687,7 @@ FieldBinding FieldBinding() throws ParseException: Expression FunctionCallExpr() throws ParseException: { Expression resultExpr; + CallExpr callExpr; List<Expression> argList = new ArrayList<Expression>(); Expression tmp = null; int arity = 0; @@ -2622,10 +2695,6 @@ Expression FunctionCallExpr() throws ParseException: String hint = null; boolean star = false; boolean distinct = false; - Token overToken = null; - Expression partitionExpr = null; - List<Expression> partitionExprs = new ArrayList<Expression>(); - OrderbyClause orderByClause = null; } { funcName = FunctionName() @@ -2664,7 +2733,7 @@ Expression FunctionCallExpr() throws ParseException: if (signature == null) { signature = new FunctionSignature(funcName.dataverse, fqFunctionName, arity); } - CallExpr callExpr = FunctionMapUtil.normalizedListInputFunctions(new CallExpr(signature,argList)); + callExpr = FunctionMapUtil.normalizedListInputFunctions(new CallExpr(signature,argList)); if (hint != null) { if (hint.startsWith(INDEXED_NESTED_LOOP_JOIN_HINT)) { callExpr.addHint(IndexedNLJoinExpressionAnnotation.INSTANCE); @@ -2676,28 +2745,171 @@ Expression FunctionCallExpr() throws ParseException: resultExpr = callExpr; } + ( <OVER> resultExpr = WindowExpr(callExpr.getFunctionSignature(), callExpr.getExprList(), token) )? + + { + return resultExpr; + } +} + +WindowExpression WindowExpr(FunctionSignature signature, List<Expression> argList, Token startToken) throws ParseException: +{ + Expression partitionExpr = null; + List<Expression> partitionExprs = new ArrayList<Expression>(); + OrderbyClause orderByClause = null; + List<Expression> orderbyList = null; + List<OrderbyClause.OrderModifier> orderbyModifierList = null; + WindowExpression.FrameMode frameMode = null; + Pair<WindowExpression.FrameBoundaryKind, Expression> frameStart = null, frameEnd = null; + WindowExpression.FrameBoundaryKind frameStartKind = null, frameEndKind = null; + Expression frameStartExpr = null, frameEndExpr = null; + WindowExpression.FrameExclusionKind frameExclusionKind = null; + Pair<VariableExpr, List<Pair<Expression, Identifier>>> windowVarWithFieldList = null; + VariableExpr windowVar = null; + List<Pair<Expression, Identifier>> windowFieldList = null; +} +{ + ( + windowVarWithFieldList = VariableWithFieldMap() <AS> + { + windowVar = windowVarWithFieldList.first; + windowFieldList = windowVarWithFieldList.second; + } + )? + <LEFTPAREN> + ( + <IDENTIFIER> { expectToken(PARTITION); } <BY> + partitionExpr = Expression() { partitionExprs.add(partitionExpr); } + ( <COMMA> partitionExpr = Expression() { partitionExprs.add(partitionExpr); } )* + )? ( - <OVER> { overToken = token; } - <LEFTPAREN> - ( - <PARTITION> <BY> - partitionExpr = Expression() { partitionExprs.add(partitionExpr); } - ( <COMMA> partitionExpr = Expression() { partitionExprs.add(partitionExpr); } )* - )? orderByClause = OrderbyClause() - <RIGHTPAREN> { - WindowExpression winExp = new WindowExpression(callExpr, partitionExprs, orderByClause.getOrderbyList(), - orderByClause.getModifierList()); - resultExpr = addSourceLocation(winExp, overToken); + orderbyList = orderByClause.getOrderbyList(); + orderbyModifierList = orderByClause.getModifierList(); } + ( + frameMode = WindowFrameMode() + ( + frameStart = WindowFrameBoundary() | + ( <BETWEEN> frameStart = WindowFrameBoundary() <AND> frameEnd = WindowFrameBoundary() ) + ) + ( frameExclusionKind = WindowFrameExclusion() )? + { + frameStartKind = frameStart.first; + frameStartExpr = frameStart.second; + if (frameEnd == null) { + frameEndKind = WindowExpression.FrameBoundaryKind.CURRENT_ROW; + } else { + frameEndKind = frameEnd.first; + frameEndExpr = frameEnd.second; + } + if (frameExclusionKind == null) { + frameExclusionKind = WindowExpression.FrameExclusionKind.NO_OTHERS; + } + } + )? )? + <RIGHTPAREN> + { + WindowExpression winExp = new WindowExpression(signature, argList, partitionExprs, orderbyList, orderbyModifierList, + frameMode, frameStartKind, frameStartExpr, frameEndKind, frameEndExpr, frameExclusionKind, windowVar, + windowFieldList); + return addSourceLocation(winExp, startToken); + } +} +WindowExpression.FrameMode WindowFrameMode() throws ParseException: +{ +} +{ + <IDENTIFIER> { - return resultExpr; + if (isToken(RANGE)) { + return WindowExpression.FrameMode.RANGE; + } else if (isToken(ROWS)) { + return WindowExpression.FrameMode.ROWS; + } else if (isToken(GROUPS)) { + return WindowExpression.FrameMode.GROUPS; + } else { + throw createUnexpectedTokenError(); + } + } +} + +Pair<WindowExpression.FrameBoundaryKind, Expression> WindowFrameBoundary() throws ParseException: +{ + boolean current = false; + Expression expr = null; +} +{ + ( + LOOKAHEAD({ laIdentifier(CURRENT) }) <IDENTIFIER> { current = true; } + | LOOKAHEAD({ laIdentifier(UNBOUNDED) }) <IDENTIFIER> + | expr = Expression() + ) + <IDENTIFIER> + { + WindowExpression.FrameBoundaryKind kind; + if (current && isToken(ROW)) { + kind = WindowExpression.FrameBoundaryKind.CURRENT_ROW; + } else if (!current && isToken(PRECEDING)) { + kind = expr == null + ? WindowExpression.FrameBoundaryKind.UNBOUNDED_PRECEDING + : WindowExpression.FrameBoundaryKind.BOUNDED_PRECEDING; + } else if (!current && isToken(FOLLOWING)) { + kind = expr == null + ? WindowExpression.FrameBoundaryKind.UNBOUNDED_FOLLOWING + : WindowExpression.FrameBoundaryKind.BOUNDED_FOLLOWING; + } else { + throw createUnexpectedTokenError(); + } + return new Pair<WindowExpression.FrameBoundaryKind, Expression>(kind, expr); } } +WindowExpression.FrameExclusionKind WindowFrameExclusion() throws ParseException: +{ + boolean current = false, no = false; +} +{ + <IDENTIFIER> + { + expectToken(EXCLUDE); + } + ( + <GROUP> + { + return WindowExpression.FrameExclusionKind.GROUP; + } + | + ( + <IDENTIFIER> + { + if (isToken(TIES)) { + return WindowExpression.FrameExclusionKind.TIES; + } else if (isToken(CURRENT)) { + current = true; + } else if (isToken(NO)) { + no = true; + } else { + throw createUnexpectedTokenError(); + } + } + <IDENTIFIER> + { + if (current && isToken(ROW)) { + return WindowExpression.FrameExclusionKind.CURRENT_ROW; + } else if (no && isToken(OTHERS)) { + return WindowExpression.FrameExclusionKind.NO_OTHERS; + } else { + throw createUnexpectedTokenError(); + } + } + ) + ) +} + Expression ParenthesizedExpression() throws ParseException: { Expression expr; @@ -3166,9 +3378,9 @@ GroupbyClause GroupbyClause()throws ParseException : Expression expr = null; VariableExpr decorVar = null; Expression decorExpr = null; - + Pair<VariableExpr, List<Pair<Expression, Identifier>>> groupVarWithFieldList = null; VariableExpr groupVar = null; - List<Pair<Expression, Identifier>> groupFieldList = new ArrayList<Pair<Expression, Identifier>>(); + List<Pair<Expression, Identifier>> groupFieldList = null; } { { @@ -3212,25 +3424,11 @@ GroupbyClause GroupbyClause()throws ParseException : } )* ) - (<GROUP> <AS> groupVar = Variable() - ( LOOKAHEAD(1) - { - VariableExpr fieldVarExpr = null; - String fieldIdentifierStr = null; - } - <LEFTPAREN> - fieldVarExpr = VariableRef() <AS> fieldIdentifierStr = Identifier() - { - groupFieldList.add(new Pair<Expression, Identifier>(fieldVarExpr, new Identifier(fieldIdentifierStr))); - } - (<COMMA> - fieldVarExpr = VariableRef() <AS> fieldIdentifierStr = Identifier() - { - groupFieldList.add(new Pair<Expression, Identifier>(fieldVarExpr, new Identifier(fieldIdentifierStr))); - } - )* - <RIGHTPAREN> - )? + (<GROUP> <AS> groupVarWithFieldList = VariableWithFieldMap() + { + groupVar = groupVarWithFieldList.first; + groupFieldList = groupVarWithFieldList.second; + } )? { gbc.setGbyPairList(vePairList); @@ -3446,7 +3644,6 @@ TOKEN [IGNORE_CASE]: | <OUTPUT : "output"> | <OVER: "over"> | <PATH : "path"> - | <PARTITION : "partition"> | <POLICY : "policy"> | <PRESORTED : "pre-sorted"> | <PRIMARY : "primary"> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/ADayTimeDuration.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/ADayTimeDuration.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/ADayTimeDuration.java index 6017d4b..7e30e2a 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/ADayTimeDuration.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/ADayTimeDuration.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.asterix.om.base.temporal.GregorianCalendarSystem; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.IAType; @@ -37,6 +38,16 @@ public class ADayTimeDuration implements IAObject { return chrononInMillisecond; } + @Override + public String toString() { + StringBuilder sbder = new StringBuilder(); + sbder.append("day_time_duration: {"); + GregorianCalendarSystem.getInstance().getDurationExtendStringRepWithTimezoneUntilField(chrononInMillisecond, 0, + sbder); + sbder.append(" }"); + return sbder.toString(); + } + /* (non-Javadoc) * @see org.apache.hyracks.api.dataflow.value.JSONSerializable#toJSON() */ http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/AYearMonthDuration.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/AYearMonthDuration.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/AYearMonthDuration.java index fd35adf..138d79c 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/AYearMonthDuration.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/AYearMonthDuration.java @@ -49,7 +49,8 @@ public class AYearMonthDuration implements IAObject { public String toString() { StringBuilder sbder = new StringBuilder(); sbder.append("year_month_duration: {"); - GregorianCalendarSystem.getInstance().getDurationMonth(chrononInMonth); + GregorianCalendarSystem.getInstance().getDurationExtendStringRepWithTimezoneUntilField(0, chrononInMonth, + sbder); sbder.append(" }"); return sbder.toString(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java index 1bb5aa9..08f5139 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java @@ -165,9 +165,12 @@ public class BuiltinFunctions { private static final Map<IFunctionInfo, IFunctionInfo> scalarToAggregateFunctionMap = new HashMap<>(); private static final Map<IFunctionInfo, IFunctionInfo> distinctToRegularScalarAggregateFunctionMap = new HashMap<>(); - private static final Map<IFunctionInfo, IFunctionInfo> builtinWindowFunctions = new HashMap<>(); - private static final Set<IFunctionInfo> builtinWindowFunctionsWithOrderArgs = new HashSet<>(); - private static final Set<IFunctionInfo> builtinWindowFunctionsWithMaterialization = new HashSet<>(); + private static final Map<IFunctionInfo, IFunctionInfo> sqlToWindowFunctions = new HashMap<>(); + private static final Set<IFunctionInfo> windowFunctions = new HashSet<>(); + private static final Set<IFunctionInfo> windowFunctionsWithListArg = new HashSet<>(); + private static final Set<IFunctionInfo> windowFunctionsWithFrameClause = new HashSet<>(); + private static final Set<IFunctionInfo> windowFunctionsWithOrderArgs = new HashSet<>(); + private static final Set<IFunctionInfo> windowFunctionsWithMaterialization = new HashSet<>(); private static final Map<IFunctionInfo, SpatialFilterKind> spatialFilterFunctions = new HashMap<>(); @@ -490,6 +493,8 @@ public class BuiltinFunctions { new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "agg-first-element", 1); public static final FunctionIdentifier LOCAL_FIRST_ELEMENT = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "agg-local-first-element", 1); + public static final FunctionIdentifier LAST_ELEMENT = + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "agg-last-element", 1); public static final FunctionIdentifier STDDEV = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "agg-stddev", 1); public static final FunctionIdentifier GLOBAL_STDDEV = @@ -538,6 +543,10 @@ public class BuiltinFunctions { new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "local-avg", 1); public static final FunctionIdentifier SCALAR_FIRST_ELEMENT = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "first-element", 1); + public static final FunctionIdentifier SCALAR_LOCAL_FIRST_ELEMENT = + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "local-first-element", 1); + public static final FunctionIdentifier SCALAR_LAST_ELEMENT = + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "last-element", 1); public static final FunctionIdentifier SCALAR_STDDEV = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "stddev", 1); public static final FunctionIdentifier SCALAR_GLOBAL_STDDEV = @@ -843,23 +852,43 @@ public class BuiltinFunctions { // window functions public static final FunctionIdentifier ROW_NUMBER = - new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "row-number", 0); + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "row_number", 0); public static final FunctionIdentifier ROW_NUMBER_IMPL = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "row-number-impl", 0); public static final FunctionIdentifier RANK = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "rank", 0); public static final FunctionIdentifier RANK_IMPL = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "rank-impl", FunctionIdentifier.VARARGS); public static final FunctionIdentifier DENSE_RANK = - new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "dense-rank", 0); + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "dense_rank", 0); public static final FunctionIdentifier DENSE_RANK_IMPL = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "dense-rank-impl", FunctionIdentifier.VARARGS); public static final FunctionIdentifier PERCENT_RANK = - new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "percent-rank", 0); + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "percent_rank", 0); public static final FunctionIdentifier PERCENT_RANK_IMPL = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "percent-rank-impl", FunctionIdentifier.VARARGS); public static final FunctionIdentifier NTILE = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "ntile", 1); public static final FunctionIdentifier NTILE_IMPL = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "ntile-impl", FunctionIdentifier.VARARGS); + public static final FunctionIdentifier LEAD = + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "lead", FunctionIdentifier.VARARGS); + public static final FunctionIdentifier LEAD_IMPL = + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "lead-impl", FunctionIdentifier.VARARGS); + public static final FunctionIdentifier LAG = + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "lag", FunctionIdentifier.VARARGS); + public static final FunctionIdentifier LAG_IMPL = + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "lag-impl", FunctionIdentifier.VARARGS); + public static final FunctionIdentifier FIRST_VALUE = + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "first_value", 1); + public static final FunctionIdentifier FIRST_VALUE_IMPL = + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "first-value-impl", 1); + public static final FunctionIdentifier LAST_VALUE = + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "last_value", 1); + public static final FunctionIdentifier LAST_VALUE_IMPL = + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "last-value-impl", 1); + public static final FunctionIdentifier NTH_VALUE = + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "nth_value", 2); + public static final FunctionIdentifier NTH_VALUE_IMPL = + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "nth-value-impl", 2); // unnesting functions public static final FunctionIdentifier SCAN_COLLECTION = @@ -1335,6 +1364,8 @@ public class BuiltinFunctions { new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "if-null", FunctionIdentifier.VARARGS); public static final FunctionIdentifier IF_MISSING_OR_NULL = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "if-missing-or-null", FunctionIdentifier.VARARGS); + public static final FunctionIdentifier IF_SYSTEM_NULL = + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "if-system-null", FunctionIdentifier.VARARGS); public static final FunctionIdentifier IF_INF = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "if-inf", FunctionIdentifier.VARARGS); public static final FunctionIdentifier IF_NAN = @@ -1420,7 +1451,7 @@ public class BuiltinFunctions { // and then, Asterix builtin functions addPrivateFunction(CHECK_UNKNOWN, NotUnknownTypeComputer.INSTANCE, true); - addPrivateFunction(ANY_COLLECTION_MEMBER, CollectionMemberResultType.INSTANCE, true); + addPrivateFunction(ANY_COLLECTION_MEMBER, CollectionMemberResultType.INSTANCE_MISSABLE, true); addFunction(BOOLEAN_CONSTRUCTOR, ABooleanTypeComputer.INSTANCE, true); addFunction(CIRCLE_CONSTRUCTOR, ACircleTypeComputer.INSTANCE, true); addPrivateFunction(CONCAT_NON_NULL, ConcatNonNullTypeComputer.INSTANCE, true); @@ -1578,6 +1609,7 @@ public class BuiltinFunctions { addFunction(IF_NULL, IfNullTypeComputer.INSTANCE, true); addFunction(IF_NAN, IfNanOrInfTypeComputer.INSTANCE_SKIP_MISSING, true); addFunction(IF_NAN_OR_INF, IfNanOrInfTypeComputer.INSTANCE_SKIP_MISSING, true); + addPrivateFunction(IF_SYSTEM_NULL, IfNullTypeComputer.INSTANCE, true); addFunction(MISSING_IF, MissingIfTypeComputer.INSTANCE, true); addFunction(NULL_IF, NullIfTypeComputer.INSTANCE, true); @@ -1595,9 +1627,12 @@ public class BuiltinFunctions { addPrivateFunction(LOCAL_AVG, LocalAvgTypeComputer.INSTANCE, true); addFunction(AVG, NullableDoubleTypeComputer.INSTANCE, true); addPrivateFunction(GLOBAL_AVG, NullableDoubleTypeComputer.INSTANCE, true); - addPrivateFunction(SCALAR_FIRST_ELEMENT, CollectionMemberResultType.INSTANCE, true); - addPrivateFunction(FIRST_ELEMENT, PropagateTypeComputer.INSTANCE, true); - addPrivateFunction(LOCAL_FIRST_ELEMENT, PropagateTypeComputer.INSTANCE, true); + addPrivateFunction(SCALAR_FIRST_ELEMENT, CollectionMemberResultType.INSTANCE_NULLABLE, true); + addPrivateFunction(SCALAR_LOCAL_FIRST_ELEMENT, CollectionMemberResultType.INSTANCE_NULLABLE, true); + addPrivateFunction(SCALAR_LAST_ELEMENT, CollectionMemberResultType.INSTANCE_NULLABLE, true); + addPrivateFunction(FIRST_ELEMENT, PropagateTypeComputer.INSTANCE_NULLABLE, true); + addPrivateFunction(LOCAL_FIRST_ELEMENT, PropagateTypeComputer.INSTANCE_NULLABLE, true); + addPrivateFunction(LAST_ELEMENT, PropagateTypeComputer.INSTANCE_NULLABLE, true); addPrivateFunction(LOCAL_STDDEV, LocalSingleVarStatisticsTypeComputer.INSTANCE, true); addFunction(STDDEV, NullableDoubleTypeComputer.INSTANCE, true); addPrivateFunction(GLOBAL_STDDEV, NullableDoubleTypeComputer.INSTANCE, true); @@ -1796,16 +1831,26 @@ public class BuiltinFunctions { // Window functions - addFunction(ROW_NUMBER, AInt64TypeComputer.INSTANCE, true); - addPrivateFunction(ROW_NUMBER_IMPL, AInt64TypeComputer.INSTANCE, true); - addFunction(RANK, AInt64TypeComputer.INSTANCE, true); - addPrivateFunction(RANK_IMPL, AInt64TypeComputer.INSTANCE, true); - addFunction(DENSE_RANK, AInt64TypeComputer.INSTANCE, true); - addPrivateFunction(DENSE_RANK_IMPL, AInt64TypeComputer.INSTANCE, true); - addFunction(PERCENT_RANK, ADoubleTypeComputer.INSTANCE, true); - addPrivateFunction(PERCENT_RANK_IMPL, ADoubleTypeComputer.INSTANCE, true); - addFunction(NTILE, AInt64TypeComputer.INSTANCE, true); - addPrivateFunction(NTILE_IMPL, AInt64TypeComputer.INSTANCE, true); + addFunction(ROW_NUMBER, AInt64TypeComputer.INSTANCE, false); + addFunction(ROW_NUMBER_IMPL, AInt64TypeComputer.INSTANCE, false); + addFunction(RANK, AInt64TypeComputer.INSTANCE, false); + addFunction(RANK_IMPL, AInt64TypeComputer.INSTANCE, false); + addFunction(DENSE_RANK, AInt64TypeComputer.INSTANCE, false); + addFunction(DENSE_RANK_IMPL, AInt64TypeComputer.INSTANCE, false); + addFunction(PERCENT_RANK, ADoubleTypeComputer.INSTANCE, false); + addFunction(PERCENT_RANK_IMPL, ADoubleTypeComputer.INSTANCE, false); + addFunction(NTILE, AInt64TypeComputer.INSTANCE_NULLABLE, false); + addFunction(NTILE_IMPL, AInt64TypeComputer.INSTANCE_NULLABLE, false); + addFunction(LEAD, AnyTypeComputer.INSTANCE, false); + addFunction(LEAD_IMPL, AnyTypeComputer.INSTANCE, false); + addFunction(LAG, AnyTypeComputer.INSTANCE, false); + addFunction(LAG_IMPL, AnyTypeComputer.INSTANCE, false); + addFunction(FIRST_VALUE, CollectionMemberResultType.INSTANCE_NULLABLE, false); + addFunction(FIRST_VALUE_IMPL, CollectionMemberResultType.INSTANCE_NULLABLE, false); + addFunction(LAST_VALUE, CollectionMemberResultType.INSTANCE_NULLABLE, false); + addFunction(LAST_VALUE_IMPL, CollectionMemberResultType.INSTANCE_NULLABLE, false); + addFunction(NTH_VALUE, CollectionMemberResultType.INSTANCE_NULLABLE, false); + addFunction(NTH_VALUE_IMPL, CollectionMemberResultType.INSTANCE_NULLABLE, false); // Similarity functions addFunction(EDIT_DISTANCE_CONTAINS, OrderedListOfAnyTypeComputer.INSTANCE, true); @@ -2261,7 +2306,7 @@ public class BuiltinFunctions { // FIRST_ELEMENT - addAgg(SCALAR_FIRST_ELEMENT); + addAgg(FIRST_ELEMENT); addAgg(LOCAL_FIRST_ELEMENT); addLocalAgg(FIRST_ELEMENT, LOCAL_FIRST_ELEMENT); addIntermediateAgg(LOCAL_FIRST_ELEMENT, FIRST_ELEMENT); @@ -2269,6 +2314,12 @@ public class BuiltinFunctions { addGlobalAgg(FIRST_ELEMENT, FIRST_ELEMENT); addScalarAgg(FIRST_ELEMENT, SCALAR_FIRST_ELEMENT); + addScalarAgg(LOCAL_FIRST_ELEMENT, SCALAR_LOCAL_FIRST_ELEMENT); + + // LAST_ELEMENT + + addAgg(LAST_ELEMENT); + addScalarAgg(LAST_ELEMENT, SCALAR_LAST_ELEMENT); // RANGE_MAP addAgg(RANGE_MAP); @@ -2574,11 +2625,16 @@ public class BuiltinFunctions { static { // Window functions - addWindowFunction(ROW_NUMBER, ROW_NUMBER_IMPL, false, false); - addWindowFunction(RANK, RANK_IMPL, true, false); - addWindowFunction(DENSE_RANK, DENSE_RANK_IMPL, true, false); - addWindowFunction(PERCENT_RANK, PERCENT_RANK_IMPL, true, true); - addWindowFunction(NTILE, NTILE_IMPL, false, true); + addWindowFunction(ROW_NUMBER, ROW_NUMBER_IMPL, false, false, false, false); + addWindowFunction(RANK, RANK_IMPL, false, false, true, false); + addWindowFunction(DENSE_RANK, DENSE_RANK_IMPL, false, false, true, false); + addWindowFunction(PERCENT_RANK, PERCENT_RANK_IMPL, false, false, true, true); + addWindowFunction(NTILE, NTILE_IMPL, false, false, false, true); + addWindowFunction(LEAD, LEAD_IMPL, false, true, false, false); + addWindowFunction(LAG, LAG_IMPL, false, true, false, false); + addWindowFunction(FIRST_VALUE, FIRST_VALUE_IMPL, true, true, false, false); + addWindowFunction(LAST_VALUE, LAST_VALUE_IMPL, true, true, false, false); + addWindowFunction(NTH_VALUE, NTH_VALUE_IMPL, true, true, false, false); } static { @@ -2762,38 +2818,59 @@ public class BuiltinFunctions { getAsterixFunctionInfo(regularscalarfi)); } - public static void addWindowFunction(FunctionIdentifier fi, FunctionIdentifier implfi, boolean requiresOrderArgs, + public static void addWindowFunction(FunctionIdentifier sqlfi, FunctionIdentifier winfi, + boolean supportsFrameClause, boolean hasListArg, boolean requiresOrderArgs, boolean requiresMaterialization) { - IFunctionInfo implFinfo = getAsterixFunctionInfo(implfi); - builtinWindowFunctions.put(getAsterixFunctionInfo(fi), implFinfo); + IFunctionInfo sqlinfo = getAsterixFunctionInfo(sqlfi); + IFunctionInfo wininfo = getAsterixFunctionInfo(winfi); + sqlToWindowFunctions.put(sqlinfo, wininfo); + windowFunctions.add(wininfo); + if (supportsFrameClause) { + windowFunctionsWithFrameClause.add(wininfo); + } + if (hasListArg) { + windowFunctionsWithListArg.add(wininfo); + } if (requiresOrderArgs) { - builtinWindowFunctionsWithOrderArgs.add(implFinfo); + windowFunctionsWithOrderArgs.add(wininfo); } if (requiresMaterialization) { - builtinWindowFunctionsWithMaterialization.add(implFinfo); + windowFunctionsWithMaterialization.add(wininfo); } } - public static boolean isBuiltinWindowFunction(FunctionIdentifier fi) { - return builtinWindowFunctions.containsKey(getAsterixFunctionInfo(fi)); + public static FunctionIdentifier getWindowFunction(FunctionIdentifier sqlfi) { + IFunctionInfo finfo = sqlToWindowFunctions.get(getAsterixFunctionInfo(sqlfi)); + return finfo == null ? null : finfo.getFunctionIdentifier(); + } + + public static boolean isWindowFunction(FunctionIdentifier winfi) { + return windowFunctions.contains(getAsterixFunctionInfo(winfi)); + } + + public static boolean windowFunctionSupportsFrameClause(FunctionIdentifier winfi) { + return windowFunctionsWithFrameClause.contains(getAsterixFunctionInfo(winfi)); + } + + public static boolean windowFunctionWithListArg(FunctionIdentifier winfi) { + return windowFunctionsWithListArg.contains(getAsterixFunctionInfo(winfi)); } - public static boolean windowFunctionRequiresOrderArgs(FunctionIdentifier implfi) { - return builtinWindowFunctionsWithOrderArgs.contains(getAsterixFunctionInfo(implfi)); + public static boolean windowFunctionRequiresOrderArgs(FunctionIdentifier winfi) { + return windowFunctionsWithOrderArgs.contains(getAsterixFunctionInfo(winfi)); } - public static boolean windowFunctionRequiresMaterialization(FunctionIdentifier implfi) { - return builtinWindowFunctionsWithMaterialization.contains(getAsterixFunctionInfo(implfi)); + public static boolean windowFunctionRequiresMaterialization(FunctionIdentifier winfi) { + return windowFunctionsWithMaterialization.contains(getAsterixFunctionInfo(winfi)); } - public static AbstractFunctionCallExpression makeWindowFunctionExpression(FunctionIdentifier scalarfi, + public static AbstractFunctionCallExpression makeWindowFunctionExpression(FunctionIdentifier winfi, List<Mutable<ILogicalExpression>> args) { - IFunctionInfo finfo = getAsterixFunctionInfo(scalarfi); - IFunctionInfo implFinfo = builtinWindowFunctions.get(finfo); - if (implFinfo == null) { + IFunctionInfo finfo = getAsterixFunctionInfo(winfi); + if (finfo == null) { throw new IllegalStateException("no implementation for window function " + finfo); } - return new StatefulFunctionCallExpression(implFinfo, UnpartitionedPropertyComputer.INSTANCE, args); + return new StatefulFunctionCallExpression(finfo, UnpartitionedPropertyComputer.INSTANCE, args); } static { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/CollectionMemberResultType.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/CollectionMemberResultType.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/CollectionMemberResultType.java index 1db946d..26bc116 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/CollectionMemberResultType.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/CollectionMemberResultType.java @@ -21,6 +21,7 @@ package org.apache.asterix.om.typecomputer.impl; import org.apache.asterix.om.exceptions.TypeMismatchException; import org.apache.asterix.om.typecomputer.base.AbstractResultTypeComputer; import org.apache.asterix.om.types.ATypeTag; +import org.apache.asterix.om.types.AUnionType; import org.apache.asterix.om.types.AbstractCollectionType; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.IAType; @@ -29,9 +30,19 @@ import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; import org.apache.hyracks.api.exceptions.SourceLocation; public class CollectionMemberResultType extends AbstractResultTypeComputer { - public static final CollectionMemberResultType INSTANCE = new CollectionMemberResultType(); + public static final CollectionMemberResultType INSTANCE = new CollectionMemberResultType(false, false); - protected CollectionMemberResultType() { + public static final CollectionMemberResultType INSTANCE_NULLABLE = new CollectionMemberResultType(true, false); + + public static final CollectionMemberResultType INSTANCE_MISSABLE = new CollectionMemberResultType(false, true); + + private final boolean nullable; + + private final boolean missable; + + private CollectionMemberResultType(boolean nullable, boolean missable) { + this.missable = missable; + this.nullable = nullable; } @Override @@ -49,7 +60,13 @@ public class CollectionMemberResultType extends AbstractResultTypeComputer { if (type.getTypeTag() == ATypeTag.ANY) { return BuiltinType.ANY; } - return ((AbstractCollectionType) type).getItemType(); + IAType itemType = ((AbstractCollectionType) type).getItemType(); + if (nullable) { + itemType = AUnionType.createNullableType(itemType); + } + if (missable) { + itemType = AUnionType.createMissableType(itemType); + } + return itemType; } - } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/PropagateTypeComputer.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/PropagateTypeComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/PropagateTypeComputer.java index df4524f..2aea87d 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/PropagateTypeComputer.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/PropagateTypeComputer.java @@ -19,6 +19,7 @@ package org.apache.asterix.om.typecomputer.impl; import org.apache.asterix.om.typecomputer.base.AbstractResultTypeComputer; +import org.apache.asterix.om.types.AUnionType; import org.apache.asterix.om.types.IAType; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; @@ -28,10 +29,19 @@ import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; */ public class PropagateTypeComputer extends AbstractResultTypeComputer { - public static final PropagateTypeComputer INSTANCE = new PropagateTypeComputer(); + public static final PropagateTypeComputer INSTANCE = new PropagateTypeComputer(false); + + public static final PropagateTypeComputer INSTANCE_NULLABLE = new PropagateTypeComputer(true); + + private final boolean nullable; + + public PropagateTypeComputer(boolean nullable) { + this.nullable = nullable; + } @Override public IAType getResultType(ILogicalExpression expr, IAType... knownTypes) throws AlgebricksException { - return knownTypes[0]; + IAType t = knownTypes[0]; + return nullable ? AUnionType.createNullableType(t) : t; } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/LastElementAggregateDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/LastElementAggregateDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/LastElementAggregateDescriptor.java new file mode 100644 index 0000000..4d2280b --- /dev/null +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/LastElementAggregateDescriptor.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.runtime.aggregates.collections; + +import org.apache.asterix.om.functions.BuiltinFunctions; +import org.apache.asterix.om.functions.IFunctionDescriptorFactory; +import org.apache.asterix.runtime.aggregates.base.AbstractAggregateFunctionDynamicDescriptor; +import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; +import org.apache.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; + +// This aggregate function simply returns the last item in an input stream. +public class LastElementAggregateDescriptor extends AbstractAggregateFunctionDynamicDescriptor { + + private static final long serialVersionUID = 1L; + public static final IFunctionDescriptorFactory FACTORY = () -> new LastElementAggregateDescriptor(); + + @Override + public FunctionIdentifier getIdentifier() { + return BuiltinFunctions.LAST_ELEMENT; + } + + @Override + public IAggregateEvaluatorFactory createAggregateEvaluatorFactory(final IScalarEvaluatorFactory[] args) { + return new LastElementEvalFactory(args, sourceLoc); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/LastElementEvalFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/LastElementEvalFactory.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/LastElementEvalFactory.java new file mode 100644 index 0000000..4b0bf35 --- /dev/null +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/LastElementEvalFactory.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.runtime.aggregates.collections; + +import org.apache.asterix.runtime.aggregates.std.AbstractAggregateFunction; +import org.apache.asterix.runtime.evaluators.functions.PointableHelper; +import org.apache.hyracks.algebricks.runtime.base.IAggregateEvaluator; +import org.apache.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; +import org.apache.hyracks.api.context.IHyracksTaskContext; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.SourceLocation; +import org.apache.hyracks.data.std.api.IPointable; +import org.apache.hyracks.data.std.primitive.VoidPointable; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference; + +class LastElementEvalFactory implements IAggregateEvaluatorFactory { + + private static final long serialVersionUID = 1L; + private final IScalarEvaluatorFactory[] args; + private final SourceLocation sourceLoc; + + LastElementEvalFactory(IScalarEvaluatorFactory[] args, SourceLocation sourceLoc) { + this.args = args; + this.sourceLoc = sourceLoc; + } + + @Override + public IAggregateEvaluator createAggregateEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException { + return new AbstractAggregateFunction(sourceLoc) { + + // Needs to copy the bytes from inputVal to outputVal because the byte space of inputVal could be re-used + // by consequent tuples. + private ArrayBackedValueStorage outputVal = new ArrayBackedValueStorage(); + private IPointable inputVal = new VoidPointable(); + private IScalarEvaluator eval = args[0].createScalarEvaluator(ctx); + private boolean empty; + + @Override + public void init() { + empty = true; + } + + @Override + public void step(IFrameTupleReference tuple) throws HyracksDataException { + eval.evaluate(tuple, inputVal); + outputVal.assign(inputVal); + empty = false; + } + + @Override + public void finish(IPointable result) { + if (empty) { + PointableHelper.setNull(result); + } else { + result.set(outputVal); + } + } + + @Override + public void finishPartial(IPointable result) { + throw new UnsupportedOperationException(); + } + }; + } + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/constructors/ATimeConstructorDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/constructors/ATimeConstructorDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/constructors/ATimeConstructorDescriptor.java index 38d8e05..9b28fb6 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/constructors/ATimeConstructorDescriptor.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/constructors/ATimeConstructorDescriptor.java @@ -94,7 +94,7 @@ public class ATimeConstructorDescriptor extends AbstractScalarFunctionDynamicDes // the string to be parsed should be at least 6 characters: hhmmss if (stringLength < 6) { throw new InvalidDataFormatException(sourceLoc, getIdentifier(), - ATypeTag.SERIALIZED_POLYGON_TYPE_TAG); + ATypeTag.SERIALIZED_TIME_TYPE_TAG); } int chrononTimeInMs = http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AbstractNumericArithmeticEval.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AbstractNumericArithmeticEval.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AbstractNumericArithmeticEval.java index 746273f..0038d58 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AbstractNumericArithmeticEval.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AbstractNumericArithmeticEval.java @@ -308,11 +308,13 @@ public abstract class AbstractNumericArithmeticEval extends AbstractScalarFuncti case FLOAT: if (evaluateDouble(operandsFloating[0], operandsFloating[1], aDouble)) { dres = aDouble.getDoubleValue(); - if (dres > Float.MAX_VALUE) { - throw new OverflowException(sourceLoc, getIdentifier()); - } - if (dres < -Float.MAX_VALUE) { - throw new UnderflowException(sourceLoc, getIdentifier()); + if (Double.isFinite(dres)) { + if (dres > Float.MAX_VALUE) { + throw new OverflowException(sourceLoc, getIdentifier()); + } + if (dres < -Float.MAX_VALUE) { + throw new UnderflowException(sourceLoc, getIdentifier()); + } } aFloat.setValue((float) dres); floatSerde.serialize(aFloat, out); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/IfSystemNullDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/IfSystemNullDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/IfSystemNullDescriptor.java new file mode 100644 index 0000000..90df330 --- /dev/null +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/IfSystemNullDescriptor.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.runtime.evaluators.functions; + +import org.apache.asterix.om.functions.BuiltinFunctions; +import org.apache.asterix.om.functions.IFunctionDescriptorFactory; +import org.apache.asterix.om.types.ATypeTag; +import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; +import org.apache.hyracks.api.context.IHyracksTaskContext; +import org.apache.hyracks.api.exceptions.HyracksDataException; + +public class IfSystemNullDescriptor extends AbstractScalarFunctionDynamicDescriptor { + private static final long serialVersionUID = 1L; + public static final IFunctionDescriptorFactory FACTORY = IfSystemNullDescriptor::new; + + @Override + public IScalarEvaluatorFactory createEvaluatorFactory(IScalarEvaluatorFactory[] args) throws AlgebricksException { + return new IScalarEvaluatorFactory() { + private static final long serialVersionUID = 1L; + + @Override + public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException { + return new IfMissingOrNullDescriptor.AbstractIfMissingOrNullEval(ctx, args) { + @Override + protected boolean skip(byte argTypeTag) { + return argTypeTag == ATypeTag.SERIALIZED_SYSTEM_NULL_TYPE_TAG; + } + }; + } + }; + } + + @Override + public FunctionIdentifier getIdentifier() { + return BuiltinFunctions.IF_SYSTEM_NULL; + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java index dceb5a8..0487385 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java @@ -29,6 +29,7 @@ import org.apache.asterix.om.functions.IFunctionCollection; import org.apache.asterix.om.functions.IFunctionDescriptorFactory; import org.apache.asterix.om.functions.IFunctionRegistrant; import org.apache.asterix.runtime.aggregates.collections.FirstElementAggregateDescriptor; +import org.apache.asterix.runtime.aggregates.collections.LastElementAggregateDescriptor; import org.apache.asterix.runtime.aggregates.collections.ListifyAggregateDescriptor; import org.apache.asterix.runtime.aggregates.collections.LocalFirstElementAggregateDescriptor; import org.apache.asterix.runtime.aggregates.scalar.ScalarAvgAggregateDescriptor; @@ -271,6 +272,7 @@ import org.apache.asterix.runtime.evaluators.functions.IfMissingOrNullDescriptor import org.apache.asterix.runtime.evaluators.functions.IfNanDescriptor; import org.apache.asterix.runtime.evaluators.functions.IfNanOrInfDescriptor; import org.apache.asterix.runtime.evaluators.functions.IfNullDescriptor; +import org.apache.asterix.runtime.evaluators.functions.IfSystemNullDescriptor; import org.apache.asterix.runtime.evaluators.functions.InjectFailureDescriptor; import org.apache.asterix.runtime.evaluators.functions.IsArrayDescriptor; import org.apache.asterix.runtime.evaluators.functions.IsAtomicDescriptor; @@ -532,6 +534,7 @@ public final class FunctionCollection implements IFunctionCollection { fc.add(LocalMinAggregateDescriptor.FACTORY); fc.add(FirstElementAggregateDescriptor.FACTORY); fc.add(LocalFirstElementAggregateDescriptor.FACTORY); + fc.add(LastElementAggregateDescriptor.FACTORY); fc.add(StddevAggregateDescriptor.FACTORY); fc.add(LocalStddevAggregateDescriptor.FACTORY); fc.add(IntermediateStddevAggregateDescriptor.FACTORY); @@ -702,6 +705,7 @@ public final class FunctionCollection implements IFunctionCollection { fc.add(IfMissingDescriptor.FACTORY); fc.add(IfNullDescriptor.FACTORY); fc.add(IfMissingOrNullDescriptor.FACTORY); + fc.add(IfSystemNullDescriptor.FACTORY); // uuid generators (zero independent functions) fc.add(CreateUUIDDescriptor.FACTORY); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/AbstractRankRunningAggregateEvaluator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/AbstractRankRunningAggregateEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/AbstractRankRunningAggregateEvaluator.java index 54bcc1a..7e6d218 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/AbstractRankRunningAggregateEvaluator.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/AbstractRankRunningAggregateEvaluator.java @@ -51,9 +51,9 @@ public abstract class AbstractRankRunningAggregateEvaluator implements IWindowAg private IBinaryComparator[] argComparators; - private boolean first; + protected boolean first; - private long rank; + protected long rank; private long groupSize; @@ -85,28 +85,28 @@ public abstract class AbstractRankRunningAggregateEvaluator implements IWindowAg @Override public void step(IFrameTupleReference tuple, IPointable result) throws HyracksDataException { + resultStorage.reset(); for (int i = 0; i < args.length; i++) { args[i].evaluate(tuple, argCurrValues[i]); } computeRank(); + computeResult(resultStorage.getDataOutput()); + + result.set(resultStorage); for (int i = 0; i < args.length; i++) { argPrevValues[i].assign(argCurrValues[i]); } - - resultStorage.reset(); - writeResult(rank, resultStorage.getDataOutput()); - result.set(resultStorage); + first = false; } - protected abstract void writeResult(long rank, DataOutput out) throws HyracksDataException; + protected abstract void computeResult(DataOutput out) throws HyracksDataException; private void computeRank() throws HyracksDataException { if (first) { rank = 1; groupSize = 1; - first = false; } else if (sameGroup()) { groupSize++; } else { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/NtileRunningAggregateEvaluator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/NtileRunningAggregateEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/NtileRunningAggregateEvaluator.java index 9bd306e..aebef6a 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/NtileRunningAggregateEvaluator.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/NtileRunningAggregateEvaluator.java @@ -22,8 +22,10 @@ package org.apache.asterix.runtime.runningaggregates.std; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; import org.apache.asterix.om.base.AInt64; import org.apache.asterix.om.base.AMutableInt64; +import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.hierachy.ATypeHierarchy; +import org.apache.asterix.runtime.evaluators.functions.PointableHelper; import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; import org.apache.hyracks.algebricks.runtime.base.IWindowAggregateEvaluator; @@ -63,6 +65,8 @@ public class NtileRunningAggregateEvaluator implements IWindowAggregateEvaluator private long count; + private boolean isNull; + NtileRunningAggregateEvaluator(IScalarEvaluator evalNumGroups, FunctionIdentifier funId) { this.evalNumGroups = evalNumGroups; this.funId = funId; @@ -76,6 +80,9 @@ public class NtileRunningAggregateEvaluator implements IWindowAggregateEvaluator public void initPartition(long partitionLength) { this.partitionLength = partitionLength; resultValue = 0; + isNull = false; + groupSize = 1; + groupRemainder = 0; } @Override @@ -93,23 +100,28 @@ public class NtileRunningAggregateEvaluator implements IWindowAggregateEvaluator count = 1; } - resultStorage.reset(); - aInt64.setValue(resultValue); - serde.serialize(aInt64, resultStorage.getDataOutput()); - result.set(resultStorage); + if (isNull) { + PointableHelper.setNull(result); + } else { + resultStorage.reset(); + aInt64.setValue(resultValue); + serde.serialize(aInt64, resultStorage.getDataOutput()); + result.set(resultStorage); + } } private void evaluateGroupSize(IFrameTupleReference tuple) throws HyracksDataException { evalNumGroups.evaluate(tuple, argNumGroups); byte[] bytes = argNumGroups.getByteArray(); int offset = argNumGroups.getStartOffset(); - long numGroups = ATypeHierarchy.getLongValue(funId.getName(), 0, bytes, offset); - if (numGroups > partitionLength || numGroups <= 0) { - groupSize = partitionLength; - groupRemainder = 0; + if (bytes[offset] == ATypeTag.SERIALIZED_NULL_TYPE_TAG) { + isNull = true; } else { - groupSize = partitionLength / numGroups; - groupRemainder = partitionLength % numGroups; + long numGroups = ATypeHierarchy.getLongValue(funId.getName(), 0, bytes, offset); + if (numGroups > 0 && numGroups <= partitionLength) { + groupSize = partitionLength / numGroups; + groupRemainder = partitionLength % numGroups; + } } } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/PercentRankRunningAggregateEvaluator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/PercentRankRunningAggregateEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/PercentRankRunningAggregateEvaluator.java index c73d9fd..edad37f 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/PercentRankRunningAggregateEvaluator.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/PercentRankRunningAggregateEvaluator.java @@ -54,8 +54,8 @@ class PercentRankRunningAggregateEvaluator extends AbstractRankRunningAggregateE } @Override - protected void writeResult(long rank, DataOutput out) throws HyracksDataException { - double percentRank = (rank - 1) / divisor; + protected void computeResult(DataOutput out) throws HyracksDataException { + double percentRank = first ? 0 : (rank - 1) / divisor; aDouble.setValue(percentRank); serde.serialize(aDouble, out); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/RankRunningAggregateEvaluator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/RankRunningAggregateEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/RankRunningAggregateEvaluator.java index 56ab299..9eba129 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/RankRunningAggregateEvaluator.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/runningaggregates/std/RankRunningAggregateEvaluator.java @@ -45,7 +45,7 @@ class RankRunningAggregateEvaluator extends AbstractRankRunningAggregateEvaluato super(args, dense, sourceLoc); } - protected void writeResult(long rank, DataOutput out) throws HyracksDataException { + protected void computeResult(DataOutput out) throws HyracksDataException { aInt64.setValue(rank); serde.serialize(aInt64, out); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/asterixdb/src/main/licenses/templates/source_only_licenses.ftl ---------------------------------------------------------------------- diff --git a/asterixdb/src/main/licenses/templates/source_only_licenses.ftl b/asterixdb/src/main/licenses/templates/source_only_licenses.ftl index eb75200..021416a 100644 --- a/asterixdb/src/main/licenses/templates/source_only_licenses.ftl +++ b/asterixdb/src/main/licenses/templates/source_only_licenses.ftl @@ -214,4 +214,30 @@ Accepting Warranty or Additional Liability. While redistributing the Work or Der END OF TERMS AND CONDITIONS </@license> +<@license files=["asterix-app/src/test/resources/runtimets/queries_sqlpp/window/pg_win/*", "asterix-app/data/tenk.tbl"] + component="AsterixDB tests" licenseName="PostgreSQL License"> +PostgreSQL Database Management System +(formerly known as Postgres, then as Postgres95) + +Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group + +Portions Copyright (c) 1994, The Regents of the University of California + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose, without fee, and without a written agreement +is hereby granted, provided that the above copyright notice and this +paragraph and the following two paragraphs appear in all copies. + +IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR +DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING +LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS +DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS +ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO +PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. +</@license> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java index 167c022..95e60a9 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java @@ -28,6 +28,11 @@ import org.apache.commons.lang3.mutable.Mutable; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan; import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; +import org.apache.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy; +import org.apache.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer; +import org.apache.hyracks.algebricks.core.algebra.typing.ITypingContext; +import org.apache.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer; +import org.apache.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment; public abstract class AbstractOperatorWithNestedPlans extends AbstractLogicalOperator { protected final List<ILogicalPlan> nestedPlans; @@ -87,6 +92,35 @@ public abstract class AbstractOperatorWithNestedPlans extends AbstractLogicalOpe return false; } + protected PropagatingTypeEnvironment createNestedPlansPropagatingTypeEnvironment(ITypingContext ctx, + boolean propagateInput) { + int n = 0; + for (ILogicalPlan p : nestedPlans) { + n += p.getRoots().size(); + } + + int i; + ITypeEnvPointer[] envPointers; + if (propagateInput) { + i = inputs.size(); + envPointers = new ITypeEnvPointer[n + i]; + for (int j = 0; j < i; j++) { + envPointers[j] = new OpRefTypeEnvPointer(inputs.get(j), ctx); + } + } else { + envPointers = new ITypeEnvPointer[n]; + i = 0; + } + for (ILogicalPlan p : nestedPlans) { + for (Mutable<ILogicalOperator> r : p.getRoots()) { + envPointers[i] = new OpRefTypeEnvPointer(r, ctx); + i++; + } + } + return new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getMissableTypeComputer(), + ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers); + } + public abstract void getUsedVariablesExceptNestedPlans(Collection<LogicalVariable> vars); public abstract void getProducedVariablesExceptNestedPlans(Collection<LogicalVariable> vars); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f2c18aa9/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java index 365d77e..49bf062 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java @@ -34,12 +34,8 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression; -import org.apache.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy; import org.apache.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy; -import org.apache.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer; import org.apache.hyracks.algebricks.core.algebra.typing.ITypingContext; -import org.apache.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer; -import org.apache.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment; import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform; import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor; @@ -233,20 +229,7 @@ public class GroupByOperator extends AbstractOperatorWithNestedPlans { @Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { - int n = 0; - for (ILogicalPlan p : nestedPlans) { - n += p.getRoots().size(); - } - ITypeEnvPointer[] envPointers = new ITypeEnvPointer[n]; - int i = 0; - for (ILogicalPlan p : nestedPlans) { - for (Mutable<ILogicalOperator> r : p.getRoots()) { - envPointers[i] = new OpRefTypeEnvPointer(r, ctx); - i++; - } - } - IVariableTypeEnvironment env = new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), - ctx.getMissableTypeComputer(), ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers); + IVariableTypeEnvironment env = createNestedPlansPropagatingTypeEnvironment(ctx, false); ILogicalOperator child = inputs.get(0).getValue(); IVariableTypeEnvironment env2 = ctx.getOutputTypeEnvironment(child); for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : getGroupByList()) {
