Author: rding Date: Wed Aug 25 00:31:38 2010 New Revision: 988770 URL: http://svn.apache.org/viewvc?rev=988770&view=rev Log: PIG-1557: couple of issue mapping aliases to jobs
Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java hadoop/pig/trunk/test/org/apache/pig/test/TestPigStats.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=988770&r1=988769&r2=988770&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Wed Aug 25 00:31:38 2010 @@ -150,6 +150,8 @@ PIG-1309: Map-side Cogroup (ashutoshc) BUG FIXES +PIG-1557: couple of issue mapping aliases to jobs (rding) + PIG-1552: Nested describe failed when the alias is not referred in the first foreach inner plan (aniket486 via daijy) PIG-1486: update ant eclipse-files target to include new jar and remove contrib dirs from build path (thejas) Modified: hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java?rev=988770&r1=988769&r2=988770&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java Wed Aug 25 00:31:38 2010 @@ -687,6 +687,11 @@ public class ScriptState { } @Override + public void visitLoad(POLoad load) throws VisitorException { + setAlias(load); + } + + @Override public void visitFRJoin(POFRJoin join) throws VisitorException { setAlias(join); } Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPigStats.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPigStats.java?rev=988770&r1=988769&r2=988770&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestPigStats.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestPigStats.java Wed Aug 25 00:31:38 2010 @@ -18,18 +18,33 @@ package org.apache.pig.test; +import static org.junit.Assert.*; + import java.io.File; import java.io.IOException; -import junit.framework.TestCase; +import junit.framework.Assert; import org.apache.pig.ExecType; +import org.apache.pig.PigException; import org.apache.pig.PigServer; import org.apache.pig.backend.executionengine.ExecJob; +import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher; +import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper; +import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan; +import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan; +import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil; +import org.apache.pig.impl.PigContext; +import org.apache.pig.impl.logicalLayer.LogicalPlan; +import org.apache.pig.impl.util.LogUtils; +import org.apache.pig.tools.pigscript.parser.ParseException; import org.apache.pig.tools.pigstats.PigStats; +import org.apache.pig.tools.pigstats.ScriptState; +import org.junit.Test; -public class TestPigStats extends TestCase { +public class TestPigStats { + @Test public void testBytesWritten_JIRA_1027() { File outputFile = null; @@ -56,6 +71,33 @@ public class TestPigStats extends TestCa } } + @Test + public void testPigStatsAlias() throws Exception { + PigServer pig = new PigServer(ExecType.LOCAL); + pig.registerQuery("A = load 'input' as (name, age, gpa);"); + pig.registerQuery("B = group A by name;"); + pig.registerQuery("C = foreach B generate group, COUNT(A);"); + pig.registerQuery("D = order C by $1;"); + pig.registerQuery("E = limit D 10;"); + pig.registerQuery("store E into 'output';"); + + LogicalPlan lp = getLogicalPlan(pig); + PhysicalPlan pp = pig.getPigContext().getExecutionEngine().compile(lp, + null); + MROperPlan mp = getMRPlan(pp, pig.getPigContext()); + + assertEquals(3, mp.getKeys().size()); + + MapReduceOper mro = mp.getRoots().get(0); + assertEquals("A,B,C", getAlias(mro)); + + mro = mp.getSuccessors(mro).get(0); + assertEquals("D", getAlias(mro)); + + mro = mp.getSuccessors(mro).get(0); + assertEquals("D,E", getAlias(mro)); + } + private void deleteDirectory( File dir ) { File[] files = dir.listFiles(); for( File file : files ) { @@ -67,4 +109,31 @@ public class TestPigStats extends TestCa } dir.delete(); } + + public static LogicalPlan getLogicalPlan(PigServer pig) throws Exception { + java.lang.reflect.Method compileLp = pig.getClass() + .getDeclaredMethod("compileLp", + new Class[] { String.class }); + compileLp.setAccessible(true); + return (LogicalPlan) compileLp.invoke(pig, new Object[] { null }); + } + + public static MROperPlan getMRPlan(PhysicalPlan pp, PigContext ctx) throws Exception { + MapReduceLauncher launcher = new MapReduceLauncher(); + java.lang.reflect.Method compile = launcher.getClass() + .getDeclaredMethod("compile", + new Class[] { PhysicalPlan.class, PigContext.class }); + compile.setAccessible(true); + return (MROperPlan) compile.invoke(launcher, new Object[] { pp, ctx }); + } + + public static String getAlias(MapReduceOper mro) throws Exception { + ScriptState ss = ScriptState.get(); + java.lang.reflect.Method getAlias = ss.getClass() + .getDeclaredMethod("getAlias", + new Class[] { MapReduceOper.class }); + getAlias.setAccessible(true); + return (String)getAlias.invoke(ss, new Object[] { mro }); + } + }