Author: gates Date: Fri May 30 09:41:59 2008 New Revision: 661785 URL: http://svn.apache.org/viewvc?rev=661785&view=rev Log: PIG-158 Santhosh's udf_fix patch.
Modified: incubator/pig/branches/types/src/org/apache/pig/data/DataType.java incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java incubator/pig/branches/types/test/org/apache/pig/test/TestSchema.java Modified: incubator/pig/branches/types/src/org/apache/pig/data/DataType.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/data/DataType.java?rev=661785&r1=661784&r2=661785&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/data/DataType.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/data/DataType.java Fri May 30 09:41:59 2008 @@ -103,7 +103,17 @@ // implements if (t instanceof Class) { Class c = (Class)t; - Class[] interfaces = c.getInterfaces(); + Class[] ioeInterfaces = c.getInterfaces(); + Class[] interfaces = null; + if(c.isInterface()){ + interfaces = new Class[ioeInterfaces.length+1]; + interfaces[0] = c; + for (int i = 1; i < interfaces.length; i++) { + interfaces[i] = ioeInterfaces[i-1]; + } + } else { + interfaces = ioeInterfaces; + } for (int i = 0; i < interfaces.length; i++) { if (interfaces[i].getName().equals("org.apache.pig.data.Tuple")) { return TUPLE; @@ -851,15 +861,11 @@ } // One is bytearray and the other is (number or chararray) - if ( (type1 == DataType.BYTEARRAY) && - ( (type2 == DataType.CHARARRAY) || (DataType.isNumberType(type2)) ) - ) { + if (type1 == DataType.BYTEARRAY) { return type2 ; } - if ( (type2 == DataType.BYTEARRAY) && - ( (type1 == DataType.CHARARRAY) || (DataType.isNumberType(type1)) ) - ) { + if (type2 == DataType.BYTEARRAY) { return type1 ; } Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=661785&r1=661784&r2=661785&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt Fri May 30 09:41:59 2008 @@ -817,7 +817,7 @@ Token t1; String funcName; List<ExpressionOperator> args; - byte type = DataType.BOOLEAN; + EvalFunc evalFunc = null; log.trace("Entering PUnaryCond"); } { @@ -917,9 +917,14 @@ log.debug("PUnaryCond: Connected operator " + cond.getClass().getName() + " " + cond + " to " + lhs + " logical plan " + lp); } ) -| LOOKAHEAD(EvalFunction(type) "(") - (funcName=EvalFunction(type) "(" args=EvalArgs(over,specs,lp,input) ")" +| LOOKAHEAD(EvalFunction() "(") + (evalFunc=EvalFunction() "(" args=EvalArgs(over,specs,lp,input) ")" { + funcName = evalFunc.getClass().getName(); + Type javaType = evalFunc.getReturnType(); + byte type = DataType.findType(javaType); + + log.debug("Return type of UDF: " + DataType.findTypeName(type)); cond = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcName, args, type); lp.add(cond); log.debug("PUnaryCond: Added operator " + cond.getClass().getName() + " " + cond + " to logical plan " + lp); @@ -1778,14 +1783,19 @@ ExpressionOperator FuncEvalSpec(Schema over, Map<String, LogicalOperator> specs, LogicalPlan lp, LogicalOperator input) : { - String funcName; + String funcName = null; List<ExpressionOperator> args; ExpressionOperator userFunc; - byte type = DataType.BYTEARRAY; + LOUserFunc userAliasFunc = null; + EvalFunc evalFunc = null; log.trace("Entering FuncEvalSpec"); } { - funcName=EvalFunction(type) "(" args=EvalArgs(over,specs,lp,input) ")" + ( + LOOKAHEAD({getDefineOp(getToken(1).image) != null}) funcName=QualifiedFunction() +| evalFunc=EvalFunction() + ) + "(" args=EvalArgs(over,specs,lp,input) ")" { //check if the function name is an alias //if the user has defined an alias then @@ -1793,13 +1803,15 @@ //and create a new LOUserFunc with the information //from the lookup table - LOUserFunc userAliasFunc = (LOUserFunc)getDefineOp(funcName); - - if(null == userAliasFunc) { - + if(null != evalFunc) { + funcName = evalFunc.getClass().getName(); + Type javaType = evalFunc.getReturnType(); + byte type = DataType.findType(javaType); + log.debug("Return type of UDF: " + DataType.findTypeName(type)); userFunc = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcName, args, type); } else { //we have an alias + userAliasFunc = (LOUserFunc)getDefineOp(funcName); int expectedNumArgs = userAliasFunc.getArguments().size(); int numArgs = args.size(); if(expectedNumArgs != numArgs) { @@ -1864,12 +1876,16 @@ List<ExpressionOperator> args; byte userSpecifiedType = DataType.BYTEARRAY; boolean userSpecified = false; - byte type = DataType.BYTEARRAY; + EvalFunc evalFunc = null; log.trace("Entering FuncDeclareSpec"); } { - [userSpecifiedType = Type() {userSpecified = true;}] funcName=EvalFunction(type) "(" args=FuncDeclareArgs(lp) ")" + [userSpecifiedType = Type() {userSpecified = true;}] evalFunc=EvalFunction() "(" args=FuncDeclareArgs(lp) ")" { + funcName = evalFunc.getClass().getName(); + Type javaType = evalFunc.getReturnType(); + byte type = DataType.findType(javaType); + log.debug("Return type of UDF: " + DataType.findTypeName(type)); if(userSpecified && (userSpecifiedType != type)) { throw new ParseException("User specified return type " + DataType.findTypeName(userSpecifiedType) + " and deduced return type " + DataType.findTypeName(type) + " are not the same"); } @@ -2096,30 +2112,26 @@ // These the simple non-terminals that are shared across many -String EvalFunction(byte type) : +EvalFunc EvalFunction() : { String funcName; + EvalFunc ef; log.trace("Entering EvalFunction"); } { funcName = QualifiedFunction() { - ExpressionOperator userFunc = getDefineOp(funcName); - if(null == userFunc) { - //TODO - //Commented out the code for instaniateFunc as it's failing - try{ - EvalFunc ef = (EvalFunc) pigContext.instantiateFuncFromAlias(funcName); - Type javaType = ((EvalFunc)ef).getReturnType(); - log.debug("Type: " + javaType); - type = DataType.findType(javaType); - }catch (Exception e){ - throw new ParseException(e.getMessage()); - } + try{ + ef = (EvalFunc) pigContext.instantiateFuncFromAlias(funcName); + Type javaType = ef.getReturnType(); + log.debug("Type: " + javaType); + log.debug("funcName: " + funcName + " class name: " + ef.getClass().getName() + " return type: " + DataType.findType(javaType)); + }catch (Exception e){ + throw new ParseException(e.getMessage()); } log.trace("Exiting EvalFunction"); - return funcName; + return ef; } } Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java?rev=661785&r1=661784&r2=661785&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java Fri May 30 09:41:59 2008 @@ -1975,10 +1975,16 @@ // flattening, we unfold field by field if (flattens.get(i)) { Schema innerSchema = leaf.getSchema() ; - for (int j=0;j < innerSchema.size(); j++) { - FieldSchema fs = innerSchema.getField(j) ; - schema.getField(outputSchemaIdx).type = fs.type ; - schema.getField(outputSchemaIdx).schema = fs.schema ; + if(null != innerSchema) { + for (int j=0;j < innerSchema.size(); j++) { + FieldSchema fs = innerSchema.getField(j) ; + schema.getField(outputSchemaIdx).type = fs.type ; + schema.getField(outputSchemaIdx).schema = fs.schema ; + outputSchemaIdx++ ; + } + } else { + schema.getField(outputSchemaIdx).type = leaf.getType() ; + schema.getField(outputSchemaIdx).schema = leaf.getSchema() ; outputSchemaIdx++ ; } } Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestSchema.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestSchema.java?rev=661785&r1=661784&r2=661785&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestSchema.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestSchema.java Fri May 30 09:41:59 2008 @@ -458,7 +458,7 @@ // Generate expected schema List<FieldSchema> expectedList = new ArrayList<FieldSchema>() ; expectedList.add(new FieldSchema("2a", DataType.BYTEARRAY)) ; - expectedList.add(new FieldSchema("2b", DataType.BYTEARRAY)) ; + expectedList.add(new FieldSchema("2b", DataType.TUPLE)) ; expectedList.add(new FieldSchema("2c", DataType.BYTEARRAY)) ; Schema expected = new Schema(expectedList) ;