Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java?rev=1447989&r1=1447988&r2=1447989&view=diff ============================================================================== --- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java (original) +++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java Wed Feb 20 02:27:17 2013 @@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.udf.pt import java.util.ArrayList; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import org.apache.hadoop.hive.ql.exec.ColumnInfo; @@ -29,20 +28,22 @@ import org.apache.hadoop.hive.ql.exec.Ex import org.apache.hadoop.hive.ql.exec.PTFPartition; import org.apache.hadoop.hive.ql.exec.PTFPartition.PTFPartitionIterator; import org.apache.hadoop.hive.ql.exec.PTFUtils; +import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.ASTNode; -import org.apache.hadoop.hive.ql.parse.PTFSpec.SelectSpec; import org.apache.hadoop.hive.ql.parse.PTFTranslator; import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.TypeCheckCtx; +import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory; +import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowExpressionSpec; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.ArgDef; +import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFExpressionDef; import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFInputDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.TableFuncDef; +import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; @@ -51,24 +52,30 @@ import org.apache.hadoop.hive.serde2.obj import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; /** - * return rows that meet a specified pattern. Use symbols to specify a list of expressions to match. - * Pattern is used to specify a Path. The results list can contain expressions based on the input columns - * and also the matched Path. + * return rows that meet a specified pattern. Use symbols to specify a list of expressions + * to match. + * Pattern is used to specify a Path. The results list can contain expressions based on + * the input columns and also the matched Path. * <ol> * <li><b>pattern:</b> pattern for the Path. Path is 'dot' separated list of symbols. - * Each element is treated as a symbol. Elements that end in '*' or '+' are interpreted with the - * usual meaning of zero or more, one or more respectively. For e.g. "LATE.EARLY*.ONTIMEOREARLY" implies a sequence of flights - * where the first occurrence was LATE, followed by zero or more EARLY flights, followed by a ONTIME or EARLY flight. + * Each element is treated as a symbol. Elements that end in '*' or '+' are interpreted with + * the usual meaning of zero or more, one or more respectively. For e.g. + * "LATE.EARLY*.ONTIMEOREARLY" implies a sequence of flights + * where the first occurrence was LATE, followed by zero or more EARLY flights, + * followed by a ONTIME or EARLY flight. * <li><b>symbols</b> specify a list of name, expression pairs. For e.g. * 'LATE', arrival_delay > 0, 'EARLY', arrival_delay < 0 , 'ONTIME', arrival_delay == 0. * These symbols can be used in the Pattern defined above. * <li><b>resultSelectList</b> specified as a select list. - * The expressions in the selectList are evaluated in the context where all the input columns are available, plus the attribute + * The expressions in the selectList are evaluated in the context where all the + * input columns are available, plus the attribute * "tpath" is available. Path is a collection of rows that represents the matching Path. * </ol> */ @@ -83,6 +90,7 @@ public class NPath extends TableFunction * the names of the Columns of the input to NPath. Used to setup the tpath Struct column. */ private ArrayList<String> inputColumnNames; + private ArrayList<String> selectListNames; @Override public void execute(PTFPartitionIterator<Object> pItr, PTFPartition outP) throws HiveException @@ -95,7 +103,8 @@ public class NPath extends TableFunction if (syFnRes.matches ) { int sz = syFnRes.nextRow - (pItr.getIndex() - 1); - Object selectListInput = NPath.getSelectListInput(iRow, tDef.getInput().getOI(), pItr, sz); + Object selectListInput = NPath.getSelectListInput(iRow, + tDef.getInput().getOutputShape().getOI(), pItr, sz); ArrayList<Object> oRow = new ArrayList<Object>(); for(ExprNodeEvaluator resExprEval : resultExprInfo.resultExprEvals) { @@ -109,14 +118,33 @@ public class NPath extends TableFunction static void throwErrorWithSignature(String message) throws SemanticException { throw new SemanticException(PTFUtils.sprintf( - "NPath signature is: SymbolPattern, one or more SymbolName, expression pairs, the result expression as a select list. Error %s", + "NPath signature is: SymbolPattern, one or more SymbolName, " + + "expression pairs, the result expression as a select list. Error %s", message)); } + + public ArrayList<String> getInputColumnNames() { + return inputColumnNames; + } + + public void setInputColumnNames(ArrayList<String> inputColumnNames) { + this.inputColumnNames = inputColumnNames; + } + + public ArrayList<String> getSelectListNames() { + return selectListNames; + } + + public void setSelectListNames(ArrayList<String> selectListNames) { + this.selectListNames = selectListNames; + } + public static class NPathResolver extends TableFunctionResolver { @Override - protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, TableFuncDef tDef) + protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, + PartitionedTableFunctionDef tDef) { return new NPath(); @@ -127,12 +155,14 @@ public class NPath extends TableFunction * <li> check structure of Arguments: * <ol> * <li> First arg should be a String - * <li> then there should be an even number of Arguments: String, expression; expression should be Convertible to Boolean. + * <li> then there should be an even number of Arguments: + * String, expression; expression should be Convertible to Boolean. * <li> finally there should be a String. * </ol> * <li> convert pattern into a NNode chain. * <li> convert symbol args into a Symbol Map. - * <li> parse selectList into SelectList struct. The inputOI used to translate these expressions should be based on the + * <li> parse selectList into SelectList struct. The inputOI used to translate + * these expressions should be based on the * columns in the Input, the 'path.attr' * </ul> */ @@ -140,9 +170,9 @@ public class NPath extends TableFunction public void setupOutputOI() throws SemanticException { NPath evaluator = (NPath) getEvaluator(); - TableFuncDef tDef = evaluator.getTableDef(); + PartitionedTableFunctionDef tDef = evaluator.getTableDef(); - ArrayList<ArgDef> args = tDef.getArgs(); + ArrayList<PTFExpressionDef> args = tDef.getArgs(); int argsNum = args == null ? 0 : args.size(); if ( argsNum < 4 ) @@ -163,7 +193,8 @@ public class NPath extends TableFunction /* * parse ResultExpr Str and setup OI. */ - ResultExpressionParser resultExprParser = new ResultExpressionParser(evaluator.resultExprStr, selectListInputRR); + ResultExpressionParser resultExprParser = + new ResultExpressionParser(evaluator.resultExprStr, selectListInputRR); try { resultExprParser.translate(); } @@ -172,55 +203,70 @@ public class NPath extends TableFunction } evaluator.resultExprInfo = resultExprParser.getResultExprInfo(); StructObjectInspector OI = evaluator.resultExprInfo.resultOI; + evaluator.selectListNames = new ArrayList<String>(); + extractOIColumnNames(resultExprParser.selectListInputOI, evaluator.selectListNames); + setOutputOI(OI); } /* * validate and setup patternStr */ - private void validateAndSetupPatternStr(NPath evaluator, ArrayList<ArgDef> args) throws SemanticException { - ArgDef symboPatternArg = args.get(0); + private void validateAndSetupPatternStr(NPath evaluator, + ArrayList<PTFExpressionDef> args) throws SemanticException { + PTFExpressionDef symboPatternArg = args.get(0); ObjectInspector symbolPatternArgOI = symboPatternArg.getOI(); if ( !ObjectInspectorUtils.isConstantObjectInspector(symbolPatternArgOI) || (symbolPatternArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) || - ((PrimitiveObjectInspector)symbolPatternArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING ) + ((PrimitiveObjectInspector)symbolPatternArgOI).getPrimitiveCategory() != + PrimitiveObjectInspector.PrimitiveCategory.STRING ) { throwErrorWithSignature("Currently the symbol Pattern must be a Constant String."); } - evaluator.patternStr = ((ConstantObjectInspector)symbolPatternArgOI).getWritableConstantValue().toString(); + evaluator.patternStr = ((ConstantObjectInspector)symbolPatternArgOI). + getWritableConstantValue().toString(); } /* * validate and setup SymbolInfo */ - private void validateAndSetupSymbolInfo(NPath evaluator, ArrayList<ArgDef> args, int argsNum) throws SemanticException { + private void validateAndSetupSymbolInfo(NPath evaluator, + ArrayList<PTFExpressionDef> args, + int argsNum) throws SemanticException { int symbolArgsSz = argsNum - 2; if ( symbolArgsSz % 2 != 0) { - throwErrorWithSignature("Symbol Name, Expression need to be specified in pairs: there are odd number of symbol args"); + throwErrorWithSignature("Symbol Name, Expression need to be specified in pairs: " + + "there are odd number of symbol args"); } evaluator.symInfo = new SymbolsInfo(symbolArgsSz/2); for(int i=1; i <= symbolArgsSz; i += 2) { - ArgDef symbolNameArg = args.get(i); + PTFExpressionDef symbolNameArg = args.get(i); ObjectInspector symbolNameArgOI = symbolNameArg.getOI(); if ( !ObjectInspectorUtils.isConstantObjectInspector(symbolNameArgOI) || (symbolNameArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) || - ((PrimitiveObjectInspector)symbolNameArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING ) + ((PrimitiveObjectInspector)symbolNameArgOI).getPrimitiveCategory() != + PrimitiveObjectInspector.PrimitiveCategory.STRING ) { - throwErrorWithSignature(PTFUtils.sprintf("Currently a Symbol Name(%s) must be a Constant String", symbolNameArg.getExpression().toStringTree())); + throwErrorWithSignature( + PTFUtils.sprintf("Currently a Symbol Name(%s) must be a Constant String", + symbolNameArg.getExpressionTreeString())); } - String symbolName = ((ConstantObjectInspector)symbolNameArgOI).getWritableConstantValue().toString(); + String symbolName = ((ConstantObjectInspector)symbolNameArgOI). + getWritableConstantValue().toString(); - ArgDef symolExprArg = args.get(i+1); + PTFExpressionDef symolExprArg = args.get(i+1); ObjectInspector symolExprArgOI = symolExprArg.getOI(); if ( (symolExprArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) || - ((PrimitiveObjectInspector)symolExprArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.BOOLEAN ) + ((PrimitiveObjectInspector)symolExprArgOI).getPrimitiveCategory() != + PrimitiveObjectInspector.PrimitiveCategory.BOOLEAN ) { - throwErrorWithSignature(PTFUtils.sprintf("Currently a Symbol Expression(%s) must be a boolean expression", symolExprArg.getExpression().toStringTree())); + throwErrorWithSignature(PTFUtils.sprintf("Currently a Symbol Expression(%s) " + + "must be a boolean expression", symolExprArg.getExpressionTreeString())); } evaluator.symInfo.add(symbolName, symolExprArg); } @@ -229,18 +275,22 @@ public class NPath extends TableFunction /* * validate and setup resultExprStr */ - private void validateAndSetupResultExprStr(NPath evaluator, ArrayList<ArgDef> args, int argsNum) throws SemanticException { - ArgDef resultExprArg = args.get(argsNum - 1); + private void validateAndSetupResultExprStr(NPath evaluator, + ArrayList<PTFExpressionDef> args, + int argsNum) throws SemanticException { + PTFExpressionDef resultExprArg = args.get(argsNum - 1); ObjectInspector resultExprArgOI = resultExprArg.getOI(); if ( !ObjectInspectorUtils.isConstantObjectInspector(resultExprArgOI) || (resultExprArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) || - ((PrimitiveObjectInspector)resultExprArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING ) + ((PrimitiveObjectInspector)resultExprArgOI).getPrimitiveCategory() != + PrimitiveObjectInspector.PrimitiveCategory.STRING ) { throwErrorWithSignature("Currently the result Expr parameter must be a Constant String."); } - evaluator.resultExprStr = ((ConstantObjectInspector)resultExprArgOI).getWritableConstantValue().toString(); + evaluator.resultExprStr = ((ConstantObjectInspector)resultExprArgOI). + getWritableConstantValue().toString(); } /* @@ -264,9 +314,9 @@ public class NPath extends TableFunction public void initializeOutputOI() throws HiveException { try { NPath evaluator = (NPath) getEvaluator(); - TableFuncDef tDef = evaluator.getTableDef(); + PartitionedTableFunctionDef tDef = evaluator.getTableDef(); - ArrayList<ArgDef> args = tDef.getArgs(); + ArrayList<PTFExpressionDef> args = tDef.getArgs(); int argsNum = args.size(); validateAndSetupPatternStr(evaluator, args); @@ -277,15 +327,16 @@ public class NPath extends TableFunction /* * setup OI for input to resultExpr select list */ - RowResolver selectListInputRR = NPath.createSelectListRR(evaluator, tDef.getInput()); - StructObjectInspector selectListInputOI = (StructObjectInspector) PTFTranslator.getInputOI(selectListInputRR); + StructObjectInspector selectListInputOI = NPath.createSelectListOI( evaluator, + tDef.getInput()); ResultExprInfo resultExprInfo = evaluator.resultExprInfo; ArrayList<ObjectInspector> selectListExprOIs = new ArrayList<ObjectInspector>(); resultExprInfo.resultExprEvals = new ArrayList<ExprNodeEvaluator>(); for(int i=0 ; i < resultExprInfo.resultExprNodes.size(); i++) { ExprNodeDesc selectColumnExprNode =resultExprInfo.resultExprNodes.get(i); - ExprNodeEvaluator selectColumnExprEval = ExprNodeEvaluatorFactory.get(selectColumnExprNode); + ExprNodeEvaluator selectColumnExprEval = + ExprNodeEvaluatorFactory.get(selectColumnExprNode); ObjectInspector selectColumnOI = selectColumnExprEval.initialize(selectListInputOI); resultExprInfo.resultExprEvals.add(selectColumnExprEval); selectListExprOIs.add(selectColumnOI); @@ -306,6 +357,15 @@ public class NPath extends TableFunction return evaluator.resultExprInfo.getResultExprNames(); } + + + private static void extractOIColumnNames(StructObjectInspector OI, + ArrayList<String> oiColumnNames) { + StructTypeInfo t = (StructTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(OI); + ArrayList<String> fnames = t.getAllStructFieldNames(); + oiColumnNames.addAll(fnames); + } + } public ResultExprInfo getResultExprInfo() { @@ -330,7 +390,7 @@ public class NPath extends TableFunction symbolExprsNames = new ArrayList<String>(sz); } - void add(String name, ArgDef arg) + void add(String name, PTFExpressionDef arg) { symbolExprsNames.add(name); symbolExprsEvaluators.add(arg.getExprEvaluator()); @@ -584,7 +644,8 @@ public class NPath extends TableFunction String symbolName = symbolNames.get(i); ExprNodeEvaluator symbolExprEval = symbolExprEvals.get(i); ObjectInspector symbolExprOI = symbolExprOIs.get(i); - symbolExprEvalMap.put(symbolName.toLowerCase(), new Object[] {symbolExprEval, symbolExprOI}); + symbolExprEvalMap.put(symbolName.toLowerCase(), + new Object[] {symbolExprEval, symbolExprOI}); } } @@ -633,7 +694,8 @@ public class NPath extends TableFunction * - the select keyword is optional. The parser checks if the expression doesn't start with * select; if not it prefixes it. * - Window Fn clauses are not permitted. - * - expressions can operate on the input columns plus the psuedo column 'path' which is array of + * - expressions can operate on the input columns plus the psuedo column 'path' + * which is array of * structs. The shape of the struct is * the same as the input. */ @@ -644,11 +706,12 @@ public class NPath extends TableFunction TypeCheckCtx selectListInputTypeCheckCtx; StructObjectInspector selectListInputOI; - SelectSpec selectSpec; + ArrayList<WindowExpressionSpec> selectSpec; ResultExprInfo resultExprInfo; - public ResultExpressionParser(String resultExprString, RowResolver selectListInputRowResolver) + public ResultExpressionParser(String resultExprString, + RowResolver selectListInputRowResolver) { this.resultExprString = resultExprString; this.selectListInputRowResolver = selectListInputRowResolver; @@ -676,15 +739,15 @@ public class NPath extends TableFunction //result ArrayList<ObjectInspector> selectListExprOIs = new ArrayList<ObjectInspector>(); int i = 0; - Iterator<Object> it = selectSpec.getColumnListAndAlias(); - while (it.hasNext()) + for(WindowExpressionSpec expr : selectSpec) { - Object[] selectColDetails = (Object[]) it.next(); - String selectColName = (String) selectColDetails[1]; - ASTNode selectColumnNode = (ASTNode) selectColDetails[2]; - ExprNodeDesc selectColumnExprNode = PTFTranslator.buildExprNode(selectColumnNode, + String selectColName = expr.getAlias(); + ASTNode selectColumnNode = expr.getExpression(); + ExprNodeDesc selectColumnExprNode = + ResultExpressionParser.buildExprNode(selectColumnNode, selectListInputTypeCheckCtx); - ExprNodeEvaluator selectColumnExprEval = ExprNodeEvaluatorFactory.get(selectColumnExprNode); + ExprNodeEvaluator selectColumnExprEval = + ExprNodeEvaluatorFactory.get(selectColumnExprNode); ObjectInspector selectColumnOI = null; selectColumnOI = selectColumnExprEval.initialize(selectListInputOI); @@ -708,7 +771,8 @@ public class NPath extends TableFunction /* * create SelectListOI */ - selectListInputOI = (StructObjectInspector) PTFTranslator.getInputOI(selectListInputRowResolver); + selectListInputOI = (StructObjectInspector) + PTFTranslator.getStandardStructOI(selectListInputRowResolver); } private void fixResultExprString() @@ -729,16 +793,9 @@ public class NPath extends TableFunction private void validateSelectExpr() throws SemanticException { - if (selectSpec.getWindowFuncs() != null) - { - throw new SemanticException( - "NPath Result Expression cannot have Windowing Function expressions"); - } - - for (ASTNode node : selectSpec.getExpressions()) + for (WindowExpressionSpec expr : selectSpec) { - PTFTranslator.validateNoLeadLagInValueBoundarySpec(node, - "Lead/Lag not allowed in NPath Result Expression"); + PTFTranslator.validateNoLeadLagInValueBoundarySpec(expr.getExpression()); } } @@ -755,6 +812,24 @@ public class NPath extends TableFunction } return "npath_col_" + colIdx; } + + public static ExprNodeDesc buildExprNode(ASTNode expr, + TypeCheckCtx typeCheckCtx) throws SemanticException + { + // todo: use SemanticAnalyzer::genExprNodeDesc + // currently SA not available to PTFTranslator. + HashMap<Node, Object> map = TypeCheckProcFactory + .genExprNode(expr, typeCheckCtx); + ExprNodeDesc desc = (ExprNodeDesc) map.get(expr); + if (desc == null) { + String errMsg = typeCheckCtx.getError(); + if ( errMsg == null) { + errMsg = "Error in parsing "; + } + throw new SemanticException(errMsg); + } + return desc; + } } public static final String PATHATTR_NAME = "tpath"; @@ -762,9 +837,10 @@ public class NPath extends TableFunction /* * add array<struct> to the list of columns */ - protected static RowResolver createSelectListRR(NPath evaluator, PTFInputDef inpDef) throws SemanticException { + protected static RowResolver createSelectListRR(NPath evaluator, + PTFInputDef inpDef) throws SemanticException { RowResolver rr = new RowResolver(); - RowResolver inputRR = inpDef.getInputInfo().getRowResolver(); + RowResolver inputRR = inpDef.getOutputShape().getRr(); boolean inputColNamesKnown = evaluator.inputColumnNames != null; if ( !inputColNamesKnown ) { @@ -797,8 +873,10 @@ public class NPath extends TableFunction inpColOIs.add(cInfo.getObjectInspector()); } - StandardListObjectInspector pathAttrOI = ObjectInspectorFactory.getStandardListObjectInspector( - ObjectInspectorFactory.getStandardStructObjectInspector(evaluator.inputColumnNames, inpColOIs)); + StandardListObjectInspector pathAttrOI = + ObjectInspectorFactory.getStandardListObjectInspector( + ObjectInspectorFactory.getStandardStructObjectInspector(evaluator.inputColumnNames, + inpColOIs)); ColumnInfo pathColumn = new ColumnInfo(PATHATTR_NAME, TypeInfoUtils.getTypeInfoFromObjectInspector(pathAttrOI), @@ -809,6 +887,25 @@ public class NPath extends TableFunction return rr; } + protected static StructObjectInspector createSelectListOI(NPath evaluator, PTFInputDef inpDef) { + StructObjectInspector inOI = inpDef.getOutputShape().getOI(); + ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); + for(StructField f : inOI.getAllStructFieldRefs()) { + fieldOIs.add(f.getFieldObjectInspector()); + } + + StandardListObjectInspector pathAttrOI = + ObjectInspectorFactory.getStandardListObjectInspector( + ObjectInspectorFactory.getStandardStructObjectInspector(evaluator.inputColumnNames, + fieldOIs)); + + ArrayList<ObjectInspector> selectFieldOIs = new ArrayList<ObjectInspector>(); + selectFieldOIs.addAll(fieldOIs); + selectFieldOIs.add(pathAttrOI); + return ObjectInspectorFactory.getStandardStructObjectInspector( + evaluator.selectListNames, selectFieldOIs); + } + public static Object getSelectListInput(Object currRow, ObjectInspector rowOI, PTFPartitionIterator<Object> pItr, int sz) { ArrayList<Object> oRow = new ArrayList<Object>();
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/Noop.java URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/Noop.java?rev=1447989&r1=1447988&r2=1447989&view=diff ============================================================================== --- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/Noop.java (original) +++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/Noop.java Wed Feb 20 02:27:17 2013 @@ -25,7 +25,7 @@ import org.apache.hadoop.hive.ql.exec.PT import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.TableFuncDef; +import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; public class Noop extends TableFunctionEvaluator @@ -47,7 +47,7 @@ public class Noop extends TableFunctionE { @Override - protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, TableFuncDef tDef) + protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef) { return new Noop(); } @@ -55,7 +55,7 @@ public class Noop extends TableFunctionE @Override public void setupOutputOI() throws SemanticException { - StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOI(); + StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI(); setOutputOI(OI); } Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java?rev=1447989&r1=1447988&r2=1447989&view=diff ============================================================================== --- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java (original) +++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java Wed Feb 20 02:27:17 2013 @@ -24,7 +24,7 @@ import org.apache.hadoop.hive.ql.exec.PT import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.TableFuncDef; +import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; public class NoopWithMap extends Noop @@ -45,7 +45,7 @@ public class NoopWithMap extends Noop { @Override - protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, TableFuncDef tDef) + protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef) { return new NoopWithMap(); } @@ -53,7 +53,7 @@ public class NoopWithMap extends Noop @Override public void setupOutputOI() throws SemanticException { - StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOI(); + StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI(); setOutputOI(OI); } @@ -80,7 +80,7 @@ public class NoopWithMap extends Noop @Override public void setupRawInputOI() throws SemanticException { - StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOI(); + StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI(); setRawInputOI(OI); } Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java?rev=1447989&r1=1447988&r2=1447989&view=diff ============================================================================== --- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java (original) +++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java Wed Feb 20 02:27:17 2013 @@ -26,7 +26,7 @@ import org.apache.hadoop.hive.ql.exec.PT import org.apache.hadoop.hive.ql.exec.PTFUtils; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.TableFuncDef; +import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -50,9 +50,18 @@ import org.apache.hadoop.hive.serde2.obj */ public abstract class TableFunctionEvaluator { + /* + * how is this different from the OutpuShape set on the TableDef. + * This is the OI of the object coming out of the PTF. + * It is put in an output Partition whose Serde is usually LazyBinarySerde. + * So the next PTF (or Operator) in the chain gets a LazyBinaryStruct. + */ transient protected StructObjectInspector OI; + /* + * same comment as OI applies here. + */ transient protected StructObjectInspector rawInputOI; - protected TableFuncDef tDef; + protected PartitionedTableFunctionDef tDef; protected PTFDesc ptfDesc; String partitionClass; int partitionMemSize; @@ -74,12 +83,12 @@ public abstract class TableFunctionEvalu OI = outputOI; } - public TableFuncDef getTableDef() + public PartitionedTableFunctionDef getTableDef() { return tDef; } - public void setTableDef(TableFuncDef tDef) + public void setTableDef(PartitionedTableFunctionDef tDef) { this.tDef = tDef; } @@ -138,7 +147,7 @@ public abstract class TableFunctionEvalu PTFPartitionIterator<Object> pItr = iPart.iterator(); PTFOperator.connectLeadLagFunctionsToPartition(ptfDesc, pItr); PTFPartition outP = new PTFPartition(getPartitionClass(), - getPartitionMemSize(), tDef.getSerde(), OI); + getPartitionMemSize(), tDef.getOutputShape().getSerde(), OI); execute(pItr, outP); return outP; } @@ -159,3 +168,4 @@ public abstract class TableFunctionEvalu return null; } } + Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java?rev=1447989&r1=1447988&r2=1447989&view=diff ============================================================================== --- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java (original) +++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java Wed Feb 20 02:27:17 2013 @@ -28,7 +28,7 @@ import org.apache.hadoop.hive.ql.metadat import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.TableFuncDef; +import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -65,11 +65,10 @@ public abstract class TableFunctionResol * - sets up the evaluator with references to the TableDef, PartitionClass, PartitonMemsize and * the transformsRawInput boolean. */ - public void initialize(PTFDesc ptfDesc, TableFuncDef tDef) + public void initialize(HiveConf cfg, PTFDesc ptfDesc, PartitionedTableFunctionDef tDef) throws SemanticException { this.ptfDesc = ptfDesc; - HiveConf cfg = ptfDesc.getTranslationInfo().getHiveCfg(); String partitionClass = HiveConf.getVar(cfg, ConfVars.HIVE_PTF_PARTITION_PERSISTENCE_CLASS); int partitionMemSize = HiveConf.getIntVar(cfg, ConfVars.HIVE_PTF_PARTITION_PERSISTENT_SIZE); @@ -85,7 +84,7 @@ public abstract class TableFunctionResol /* * called during deserialization of a QueryDef during runtime. */ - public void initialize(PTFDesc ptfDesc, TableFuncDef tDef, TableFunctionEvaluator evaluator) + public void initialize(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef, TableFunctionEvaluator evaluator) throws HiveException { this.evaluator = evaluator; @@ -211,6 +210,5 @@ public abstract class TableFunctionResol /* * a subclass must provide the {@link TableFunctionEvaluator} instance. */ - protected abstract TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, - TableFuncDef tDef); + protected abstract TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef); } Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java?rev=1447989&r1=1447988&r2=1447989&view=diff ============================================================================== --- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java (original) +++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java Wed Feb 20 02:27:17 2013 @@ -22,32 +22,25 @@ import java.util.AbstractList; import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.PTFOperator; import org.apache.hadoop.hive.ql.exec.PTFPartition; import org.apache.hadoop.hive.ql.exec.PTFPartition.PTFPartitionIterator; -import org.apache.hadoop.hive.ql.exec.WindowFunctionInfo; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.BoundarySpec; -import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.Direction; -import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFunctionSpec; -import org.apache.hadoop.hive.ql.parse.PTFTranslator; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.WindowingSpec.BoundarySpec; +import org.apache.hadoop.hive.ql.parse.WindowingSpec.Direction; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.ArgDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.SelectDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.TableFuncDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFrameDef.BoundaryDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFrameDef.CurrentRowDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFrameDef.RangeBoundaryDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFrameDef.ValueBoundaryDef; +import org.apache.hadoop.hive.ql.plan.PTFDesc.BoundaryDef; +import org.apache.hadoop.hive.ql.plan.PTFDesc.CurrentRowDef; +import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFExpressionDef; +import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.PTFDesc.RangeBoundaryDef; +import org.apache.hadoop.hive.ql.plan.PTFDesc.ValueBoundaryDef; import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFunctionDef; +import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowTableFunctionDef; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer; import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -56,7 +49,19 @@ import org.apache.hadoop.hive.serde2.obj public class WindowingTableFunction extends TableFunctionEvaluator { - ArrayList<WindowFunctionDef> wFnDefs; + + @Override + public PTFPartition execute(PTFPartition iPart) + throws HiveException + { + WindowTableFunctionDef wFnDef = (WindowTableFunctionDef) getTableDef(); + PTFPartitionIterator<Object> pItr = iPart.iterator(); + PTFOperator.connectLeadLagFunctionsToPartition(ptfDesc, pItr); + PTFPartition outP = new PTFPartition(getPartitionClass(), + getPartitionMemSize(), wFnDef.getOutputFromWdwFnProcessing().getSerde(), OI); + execute(pItr, outP); + return outP; + } @SuppressWarnings({ "unchecked", "rawtypes" }) @Override @@ -71,13 +76,15 @@ public class WindowingTableFunction exte throw new HiveException(se); } - for(WindowFunctionDef wFn : wFnDefs) + WindowTableFunctionDef wTFnDef = (WindowTableFunctionDef) getTableDef(); + + for(WindowFunctionDef wFn : wTFnDef.getWindowFunctions()) { - boolean processWindow = wFn.getWindow() != null && wFn.getWindow().getWindow() != null; + boolean processWindow = wFn.getWindowFrame() != null; pItr.reset(); if ( !processWindow ) { - GenericUDAFEvaluator fEval = wFn.getEvaluator(); + GenericUDAFEvaluator fEval = wFn.getwFnEval(); Object[] args = new Object[wFn.getArgs() == null ? 0 : wFn.getArgs().size()]; AggregationBuffer aggBuffer = fEval.getNewAggregationBuffer(); while(pItr.hasNext()) @@ -85,7 +92,7 @@ public class WindowingTableFunction exte Object row = pItr.next(); int i =0; if ( wFn.getArgs() != null ) { - for(ArgDef arg : wFn.getArgs()) + for(PTFExpressionDef arg : wFn.getArgs()) { args[i++] = arg.getExprEvaluator().evaluate(row); } @@ -93,8 +100,7 @@ public class WindowingTableFunction exte fEval.aggregate(aggBuffer, args); } Object out = fEval.evaluate(aggBuffer); - WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFn.getSpec().getName()); - if ( !wFnInfo.isPivotResult()) + if ( !wFn.isPivotResult()) { out = new SameList(iPart.size(), out); } @@ -133,58 +139,30 @@ public class WindowingTableFunction exte public static class WindowingTableFunctionResolver extends TableFunctionResolver { + /* + * OI of object constructed from output of Wdw Fns; before it is put + * in the Wdw Processing Partition. Set by Translator/Deserializer. + */ + private transient StructObjectInspector wdwProcessingOutputOI; - @Override - protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, TableFuncDef tDef) - { + public StructObjectInspector getWdwProcessingOutputOI() { + return wdwProcessingOutputOI; + } - return new WindowingTableFunction(); + public void setWdwProcessingOutputOI(StructObjectInspector wdwProcessingOutputOI) { + this.wdwProcessingOutputOI = wdwProcessingOutputOI; } @Override - public void setupOutputOI() throws SemanticException + protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef) { - ArrayList<WindowFunctionDef> wFnDefs = new ArrayList<WindowFunctionDef>(); - PTFDesc ptfDesc = getPtfDesc(); - SelectDef select = ptfDesc.getSelectList(); - ArrayList<WindowFunctionSpec> wFnSpecs = ptfDesc.getSpec().getSelectList().getWindowFuncs(); - ArrayList<String> aliases = new ArrayList<String>(); - ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); - - /* - * Setup the columns in the OI in the following order - * - the columns representing the Window Fns - * - the columns from the input - * Why? - * - during translation the input contains Virtual columns that are not represent during runtime - * - this messes with the Column Numbers (and hence internal Names) if we add the columns in a different order. - */ - - for(WindowFunctionSpec wFnS : wFnSpecs) - { - WindowFunctionDef wFnDef = PTFTranslator.translate(ptfDesc, getEvaluator().getTableDef(), wFnS); - WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFnS.getName()); - wFnDefs.add(wFnDef); - aliases.add(wFnS.getAlias()); - if ( wFnInfo.isPivotResult()) - { - ListObjectInspector lOI = (ListObjectInspector) wFnDef.getOI(); - fieldOIs.add(lOI.getListElementObjectInspector()); - } - else - { - fieldOIs.add(wFnDef.getOI()); - } - } - - PTFTranslator.addInputColumnsToList(ptfDesc, getEvaluator().getTableDef(), aliases, fieldOIs); - select.setWindowFuncs(wFnDefs); - WindowingTableFunction wTFn = (WindowingTableFunction) getEvaluator(); - wTFn.wFnDefs = wFnDefs; + return new WindowingTableFunction(); + } - StructObjectInspector OI = ObjectInspectorFactory.getStandardStructObjectInspector(aliases, fieldOIs); - setOutputOI(OI); + @Override + public void setupOutputOI() throws SemanticException { + setOutputOI(wdwProcessingOutputOI); } /* @@ -195,42 +173,7 @@ public class WindowingTableFunction exte @Override public void initializeOutputOI() throws HiveException { - PTFDesc ptfDesc = getPtfDesc(); - TableFuncDef tblFuncDef = evaluator.getTableDef(); - WindowingTableFunction wTFn = (WindowingTableFunction) tblFuncDef.getFunction(); - ArrayList<WindowFunctionDef> wFnDefs = ptfDesc.getSelectList().getWindowFuncs(); - ArrayList<String> aliases = new ArrayList<String>(); - ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); - - /* - * Setup the columns in the OI in the following order - * - the columns representing the Window Fns - * - the columns from the input - * Why? - * - during translation the input contains Virtual columns that are not present during runtime - * - this messes with the Column Numbers (and hence internal Names) if we add the columns in a different order. - */ - - for (WindowFunctionDef wFnDef : wFnDefs) { - WindowFunctionSpec wFnS = wFnDef.getSpec(); - WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFnS.getName()); - aliases.add(wFnS.getAlias()); - if ( wFnInfo.isPivotResult()) - { - ListObjectInspector lOI = (ListObjectInspector) wFnDef.getOI(); - fieldOIs.add(lOI.getListElementObjectInspector()); - } - else - { - fieldOIs.add(wFnDef.getOI()); - } - - } - PTFTranslator.addInputColumnsToList(ptfDesc, getEvaluator().getTableDef(), aliases, fieldOIs); - - wTFn.wFnDefs = wFnDefs; - StructObjectInspector OI = ObjectInspectorFactory.getStandardStructObjectInspector(aliases, fieldOIs); - setOutputOI(OI); + setupOutputOI(); } @@ -267,7 +210,7 @@ public class WindowingTableFunction exte { ArrayList<Object> vals = new ArrayList<Object>(); - GenericUDAFEvaluator fEval = wFnDef.getEvaluator(); + GenericUDAFEvaluator fEval = wFnDef.getwFnEval(); Object[] args = new Object[wFnDef.getArgs().size()]; for(int i=0; i < iPart.size(); i++) { @@ -279,7 +222,7 @@ public class WindowingTableFunction exte { Object row = rItr.next(); int j = 0; - for(ArgDef arg : wFnDef.getArgs()) + for(PTFExpressionDef arg : wFnDef.getArgs()) { args[j++] = arg.getExprEvaluator().evaluate(row); } @@ -294,8 +237,8 @@ public class WindowingTableFunction exte static Range getRange(WindowFunctionDef wFnDef, int currRow, PTFPartition p) throws HiveException { - BoundaryDef startB = wFnDef.getWindow().getWindow().getStart(); - BoundaryDef endB = wFnDef.getWindow().getWindow().getEnd(); + BoundaryDef startB = wFnDef.getWindowFrame().getStart(); + BoundaryDef endB = wFnDef.getWindowFrame().getEnd(); int start = getIndex(startB, currRow, p, false); int end = getIndex(endB, currRow, p, true); @@ -440,6 +383,7 @@ public class WindowingTableFunction exte public abstract boolean isGreater(Object v1, Object v2, int amt); + @SuppressWarnings("incomplete-switch") public static ValueBoundaryScanner getScanner(ValueBoundaryDef vbDef) { PrimitiveObjectInspector pOI = (PrimitiveObjectInspector) vbDef.getOI(); Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_general_queries.q URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_general_queries.q?rev=1447989&r1=1447988&r2=1447989&view=diff ============================================================================== --- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_general_queries.q (original) +++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_general_queries.q Wed Feb 20 02:27:17 2013 @@ -32,7 +32,7 @@ LOAD DATA LOCAL INPATH '../data/files/fl select p_mfgr, p_name, p_size, rank() as r, dense_rank() as dr, -sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +sum(p_retailprice) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) from noop(part distribute by p_mfgr sort by p_name @@ -51,7 +51,7 @@ sort by p_name; select p_mfgr, p_name, rank() as r, dense_rank() as dr, -p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row), +p_retailprice, sum(p_retailprice) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row), p_size, p_size - lag(p_size,1) as deltaSz from noop(part distribute by p_mfgr @@ -74,19 +74,14 @@ from part distribute by p_mfgr sort by p_name ; --- 6. testJoinWithLeadLag -select p1.p_mfgr, p1.p_name, -p1.p_size, p1.p_size - lag(p1.p_size,1) as deltaSz -from part p1 join part p2 on p1.p_partkey = p2.p_partkey -distribute by p1.p_mfgr -sort by p1.p_name ; - -- 7. testJoinWithNoop select p_mfgr, p_name, p_size, p_size - lag(p_size,1) as deltaSz from noop ( (select p1.* from part p1 join part p2 on p1.p_partkey = p2.p_partkey) j distribute by j.p_mfgr -sort by j.p_name); +sort by j.p_name) +distribute by p_mfgr +sort by p_name; -- 8. testGroupByWithSWQ select p_mfgr, p_name, p_size, min(p_retailprice), @@ -119,7 +114,7 @@ sort by p_name); select p_mfgr, p_name, p_size, rank() as r, dense_rank() as dr, -sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +sum(p_retailprice) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) from noop(part distribute by p_mfgr sort by p_name @@ -156,9 +151,9 @@ select p_mfgr, p_name, rank() as r, dense_rank() as dr, count(p_size) as cd -from noop(part +from part distribute by p_mfgr -sort by p_name); +sort by p_name; -- 15. testCountWithWindowing select p_mfgr, p_name, @@ -167,9 +162,9 @@ dense_rank() as dr, count(p_size) as cd, p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row), p_size, p_size - lag(p_size,1) as deltaSz -from noop(part +from part distribute by p_mfgr -sort by p_name); +sort by p_name; -- 16. testCountInSubQ @@ -180,9 +175,9 @@ dense_rank() as dr, count(p_size) as cd, p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row), p_size, p_size - lag(p_size,1) as deltaSz -from noop(part +from part distribute by p_mfgr -sort by p_name) +sort by p_name ) sub1; -- 17. testJoin @@ -214,12 +209,13 @@ sort by abc.p_name ; -- 20. testMixedCaseAlias select p_mfgr, p_name, p_size, rank() as R -from noop(part +from part distribute by p_mfgr -sort by p_name, p_size desc); +sort by p_name, p_size desc; -- 21. testNoopWithMap -select p_mfgr, p_name, p_size, rank() as r +select p_mfgr, p_name, p_size, +rank() as r over (distribute by p_mfgr sort by p_name, p_size desc) from noopwithmap(part distribute by p_mfgr sort by p_name, p_size desc); @@ -228,7 +224,7 @@ sort by p_name, p_size desc); select p_mfgr, p_name, p_size, rank() as r, dense_rank() as dr, -sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +sum(p_retailprice) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) from noopwithmap(part distribute by p_mfgr sort by p_name); @@ -242,14 +238,14 @@ from part having p_size > 5 distribute by p_mfgr sort by p_name; - + -- 24. testHavingWithWindowingCondRankNoGBY select p_mfgr, p_name, p_size, rank() as r, dense_rank() as dr, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) from part -having r < 4 +having rank() < 4 distribute by p_mfgr sort by p_name; @@ -257,12 +253,12 @@ sort by p_name; select p_mfgr, p_name, p_size, rank() as r, dense_rank() as dr, -sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +sum(p_retailprice) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) from noop(part distribute by p_mfgr sort by p_name) -having r < 4; - +having rank() < 4; + -- 26. testFirstLast select p_mfgr,p_name, p_size, sum(p_size) as s2 over (rows between current row and current row), @@ -345,7 +341,7 @@ window w1 as (rows between 2 preceding a select p_mfgr, p_name, p_size, rank() as r, dense_rank() as dr, -sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +sum(p_retailprice) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) from noop(noopwithmap(noop(part distribute by p_mfgr sort by p_mfgr, p_name @@ -361,7 +357,7 @@ sum(p_retailprice) as s1 over w1 from noop(part distribute by p_mfgr sort by p_name) -window w1 as (rows between 2 preceding and 2 following) +window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following) ) sub1 ; -- 35. testCountStar @@ -479,7 +475,7 @@ sum(s) as s1 over w1 from noop(mfgr_price_view distribute by p_mfgr sort by p_mfgr) -window w1 as (rows between 2 preceding and current row); +window w1 as ( distribute by p_mfgr sort by p_brand rows between 2 preceding and current row); -- 46. testCreateViewWithWindowingQuery create view IF NOT EXISTS mfgr_brand_price_view as @@ -566,7 +562,7 @@ select * from part_1; select * from part_2; select * from part_3; - + -- 50. testGroupByHavingWithSWQAndAlias select p_mfgr, p_name, p_size, min(p_retailprice) as mi, rank() as r, @@ -577,7 +573,7 @@ group by p_mfgr, p_name, p_size having p_size > 0 distribute by p_mfgr sort by p_name; - + -- 51. testMultipleRangeWindows select p_mfgr,p_name, p_size, sum(p_size) as s2 over (range between p_size 10 less and current row), @@ -707,7 +703,7 @@ window w1 as (rows between 2 preceding a select p_mfgr, p_name, rank() as r, dense_rank() as dr, -p_size, sum(p_size) as s1 over (rows between unbounded preceding and current row) +p_size, sum(p_size) as s1 over (distribute by p_mfgr,p_name rows between unbounded preceding and current row) from noop( noopwithmap( noop( @@ -724,7 +720,7 @@ from noop( select p_mfgr, p_name, rank() as r, dense_rank() as dr, -p_size, sum(p_size) as s1 over (rows between unbounded preceding and current row) +p_size, sum(p_size) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) from noop( noop( noop( @@ -739,7 +735,7 @@ from noop( -- 63. testMultiOperatorChainWithNoWindowing select p_mfgr, p_name, -rank() as r, +rank() as r over (distribute by p_mfgr sort by p_name), dense_rank() as dr, p_size, sum(p_size) as s1 from noop( @@ -757,7 +753,7 @@ from noop( select p_mfgr, p_name, rank() as r, dense_rank() as dr, -p_size, sum(p_size) as s1 over (rows between unbounded preceding and current row) +p_size, sum(p_size) as s1 over (distribute by p_mfgr,p_name rows between unbounded preceding and current row) from noopwithmap( noop( noop( Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_rcfile.q URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_rcfile.q?rev=1447989&r1=1447988&r2=1447989&view=diff ============================================================================== --- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_rcfile.q (original) +++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_rcfile.q Wed Feb 20 02:27:17 2013 @@ -18,7 +18,7 @@ LOAD DATA LOCAL INPATH '../data/files/pa select p_mfgr, p_name, p_size, rank() as r, dense_rank() as dr, -sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +sum(p_retailprice) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) from noop(part_rc distribute by p_mfgr sort by p_name); Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_seqfile.q URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_seqfile.q?rev=1447989&r1=1447988&r2=1447989&view=diff ============================================================================== --- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_seqfile.q (original) +++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_seqfile.q Wed Feb 20 02:27:17 2013 @@ -18,7 +18,7 @@ LOAD DATA LOCAL INPATH '../data/files/pa select p_mfgr, p_name, p_size, rank() as r, dense_rank() as dr, -sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) +sum(p_retailprice) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) from noop(part_seq distribute by p_mfgr sort by p_name);
