Author: dvryaboy Date: Mon May 3 19:01:38 2010 New Revision: 940570 URL: http://svn.apache.org/viewvc?rev=940570&view=rev Log: PIG-1303: Inconsistent instantiation of parametrized UDFs (jrussek and dvryaboy)
Added: hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestAlgebraicInstantiation.java Modified: hadoop/pig/branches/branch-0.7/CHANGES.txt hadoop/pig/branches/branch-0.7/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java Modified: hadoop/pig/branches/branch-0.7/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/CHANGES.txt?rev=940570&r1=940569&r2=940570&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.7/CHANGES.txt (original) +++ hadoop/pig/branches/branch-0.7/CHANGES.txt Mon May 3 19:01:38 2010 @@ -187,6 +187,8 @@ OPTIMIZATIONS BUG FIXES +PIG-1303: Inconsistent instantiation of parametrized UDFs (jrussek and dvryaboy) + PIG-1348: PigStorage making unnecessary byte array copy when storing data (rding) Modified: hadoop/pig/branches/branch-0.7/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java?rev=940570&r1=940569&r2=940570&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.7/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java (original) +++ hadoop/pig/branches/branch-0.7/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java Mon May 3 19:01:38 2010 @@ -32,22 +32,19 @@ import org.apache.pig.EvalFunc; import org.apache.pig.FuncSpec; import org.apache.pig.PigException; import org.apache.pig.backend.executionengine.ExecException; +import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus; +import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; +import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result; +import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor; import org.apache.pig.data.DataBag; import org.apache.pig.data.DataByteArray; import org.apache.pig.data.DataType; import org.apache.pig.data.Tuple; import org.apache.pig.data.TupleFactory; import org.apache.pig.impl.PigContext; -import org.apache.pig.impl.plan.OperatorKey; import org.apache.pig.impl.logicalLayer.schema.Schema; -import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigHadoopLogger; -import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.ProgressableReporter; -import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus; -import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; -import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result; -import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor; -import org.apache.pig.impl.plan.OperatorKey; import org.apache.pig.impl.plan.NodeIdGenerator; +import org.apache.pig.impl.plan.OperatorKey; import org.apache.pig.impl.plan.VisitorException; public class POUserFunc extends ExpressionOperator { @@ -324,8 +321,8 @@ public class POUserFunc extends Expressi case FINAL: funcSpec = new FuncSpec(getFinal()); break; - } + funcSpec.setCtorArgs(origFSpec.getCtorArgs()); instantiateFunc(funcSpec); setResultType(DataType.findType(((EvalFunc<?>) func).getReturnType())); } Modified: hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java?rev=940570&r1=940569&r2=940570&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java (original) +++ hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java Mon May 3 19:01:38 2010 @@ -1190,6 +1190,7 @@ public class TypeCheckingVisitor extends "different input argument types, please use explicit casts."; msgCollector.collect(msg, MessageType.Warning, PigWarning.USING_OVERLOADED_FUNCTION); } + matchingSpec.setCtorArgs(func.getFuncSpec().getCtorArgs()); func.setFuncSpec(matchingSpec); insertCastsForUDF(func, s, matchingSpec.getInputArgsSchema()); Added: hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestAlgebraicInstantiation.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestAlgebraicInstantiation.java?rev=940570&view=auto ============================================================================== --- hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestAlgebraicInstantiation.java (added) +++ hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestAlgebraicInstantiation.java Mon May 3 19:01:38 2010 @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.pig.test; + +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.Iterator; + +import org.apache.pig.Algebraic; +import org.apache.pig.EvalFunc; +import org.apache.pig.ExecType; +import org.apache.pig.PigServer; +import org.apache.pig.backend.executionengine.ExecException; +import org.apache.pig.data.Tuple; +import org.apache.pig.data.TupleFactory; +import org.junit.Before; +import org.junit.Test; + +public class TestAlgebraicInstantiation { + + Boolean[] nullFlags = new Boolean[]{ false, true}; + MiniCluster cluster = MiniCluster.buildCluster(); + private PigServer pig; + private File tmpFile; + + public TestAlgebraicInstantiation() throws ExecException { + pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties()); + } + + @Before + public void setUp() throws Exception { + + tmpFile = File.createTempFile("test", "txt"); + PrintStream ps = new PrintStream(new FileOutputStream(tmpFile)); + ps.println("1\t2"); + ps.close(); + } + + + @Test + public void testAlgebraicInstantiation() throws IOException { + pig.registerQuery("a = group (load '" + + Util.generateURI(tmpFile.toString(), pig.getPigContext()) + "') by ($0);"); + pig.registerQuery("b = foreach a generate org.apache.pig.test.TestAlgebraicInstantiation$AlgInstUDF($1.$1);"); + Iterator<Tuple> tupIter = pig.openIterator("b"); + assertEquals("no-args", tupIter.next().toDelimitedString(",")); + pig.registerQuery("DEFINE instantiated org.apache.pig.test.TestAlgebraicInstantiation$AlgInstUDF('args');"); + pig.registerQuery("b = foreach a generate instantiated($1.$1);"); + tupIter = pig.openIterator("b"); + assertEquals("args", tupIter.next().toDelimitedString(",")); + } + + @Test + public void testRegularInstantiation() throws IOException { + pig.registerQuery("a = group (load '" + + Util.generateURI(tmpFile.toString(), pig.getPigContext()) + "') by ($0);"); + pig.registerQuery("b = foreach a generate org.apache.pig.test.TestAlgebraicInstantiation$ParamUDF($1.$1);"); + Iterator<Tuple> tupIter = pig.openIterator("b"); + assertEquals("no-args", tupIter.next().toDelimitedString(",")); + pig.registerQuery("DEFINE instantiated org.apache.pig.test.TestAlgebraicInstantiation$ParamUDF('args');"); + pig.registerQuery("b = foreach a generate instantiated($1.$1);"); + tupIter = pig.openIterator("b"); + assertEquals("args", tupIter.next().toDelimitedString(",")); + } + + public static class ParamUDF extends EvalFunc<String> { + private String initType; + + public ParamUDF() { + initType = "no-args"; + } + public ParamUDF(String s) { + initType = s; + } + + @Override + public String exec(Tuple input) { + return initType; + } + } + + public static class AlgInstUDF extends EvalFunc<String> implements Algebraic { + + private String initType; + + public AlgInstUDF() { + initType = "no-args"; + } + + public AlgInstUDF(String s) { + initType = s; + } + + @Override + public String exec(Tuple input) throws IOException { + // TODO Auto-generated method stub + return initType; + } + + @Override + public String getFinal() { + return Final.class.getName(); + } + + @Override + public String getInitial() { + return Initial.class.getName(); + } + + @Override + public String getIntermed() { + return Intermed.class.getName(); + } + + public static class Initial extends EvalFunc<Tuple> { + + private String initType; + + public Initial() { + initType = "no-args"; + } + + public Initial(String s) { + initType = s; + } + + @Override + public Tuple exec(Tuple input) { + return TupleFactory.getInstance().newTuple(initType); + } + } + + public static class Intermed extends EvalFunc<Tuple> { + + private String initType; + + public Intermed() { + initType = "no-args"; + } + + public Intermed(String s) { + initType = s; + } + + @Override + public Tuple exec(Tuple input) { + return TupleFactory.getInstance().newTuple(initType); + } + } + + public static class Final extends EvalFunc<String> { + + private String initType; + + public Final() { + initType = "no-args"; + } + + public Final(String s) { + initType = s; + } + + @Override + public String exec(Tuple input) { + return initType; + } + } + } +}