Author: rding
Date: Wed Aug 25 00:31:38 2010
New Revision: 988770
URL: http://svn.apache.org/viewvc?rev=988770&view=rev
Log:
PIG-1557: couple of issue mapping aliases to jobs
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
hadoop/pig/trunk/test/org/apache/pig/test/TestPigStats.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=988770&r1=988769&r2=988770&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Wed Aug 25 00:31:38 2010
@@ -150,6 +150,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1557: couple of issue mapping aliases to jobs (rding)
+
PIG-1552: Nested describe failed when the alias is not referred in the first
foreach inner plan (aniket486 via daijy)
PIG-1486: update ant eclipse-files target to include new jar and remove
contrib dirs from build path (thejas)
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java?rev=988770&r1=988769&r2=988770&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java Wed Aug
25 00:31:38 2010
@@ -687,6 +687,11 @@ public class ScriptState {
}
@Override
+ public void visitLoad(POLoad load) throws VisitorException {
+ setAlias(load);
+ }
+
+ @Override
public void visitFRJoin(POFRJoin join) throws VisitorException {
setAlias(join);
}
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPigStats.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPigStats.java?rev=988770&r1=988769&r2=988770&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPigStats.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPigStats.java Wed Aug 25
00:31:38 2010
@@ -18,18 +18,33 @@
package org.apache.pig.test;
+import static org.junit.Assert.*;
+
import java.io.File;
import java.io.IOException;
-import junit.framework.TestCase;
+import junit.framework.Assert;
import org.apache.pig.ExecType;
+import org.apache.pig.PigException;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecJob;
+import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher;
+import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper;
+import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
+import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.logicalLayer.LogicalPlan;
+import org.apache.pig.impl.util.LogUtils;
+import org.apache.pig.tools.pigscript.parser.ParseException;
import org.apache.pig.tools.pigstats.PigStats;
+import org.apache.pig.tools.pigstats.ScriptState;
+import org.junit.Test;
-public class TestPigStats extends TestCase {
+public class TestPigStats {
+ @Test
public void testBytesWritten_JIRA_1027() {
File outputFile = null;
@@ -56,6 +71,33 @@ public class TestPigStats extends TestCa
}
}
+ @Test
+ public void testPigStatsAlias() throws Exception {
+ PigServer pig = new PigServer(ExecType.LOCAL);
+ pig.registerQuery("A = load 'input' as (name, age, gpa);");
+ pig.registerQuery("B = group A by name;");
+ pig.registerQuery("C = foreach B generate group, COUNT(A);");
+ pig.registerQuery("D = order C by $1;");
+ pig.registerQuery("E = limit D 10;");
+ pig.registerQuery("store E into 'output';");
+
+ LogicalPlan lp = getLogicalPlan(pig);
+ PhysicalPlan pp = pig.getPigContext().getExecutionEngine().compile(lp,
+ null);
+ MROperPlan mp = getMRPlan(pp, pig.getPigContext());
+
+ assertEquals(3, mp.getKeys().size());
+
+ MapReduceOper mro = mp.getRoots().get(0);
+ assertEquals("A,B,C", getAlias(mro));
+
+ mro = mp.getSuccessors(mro).get(0);
+ assertEquals("D", getAlias(mro));
+
+ mro = mp.getSuccessors(mro).get(0);
+ assertEquals("D,E", getAlias(mro));
+ }
+
private void deleteDirectory( File dir ) {
File[] files = dir.listFiles();
for( File file : files ) {
@@ -67,4 +109,31 @@ public class TestPigStats extends TestCa
}
dir.delete();
}
+
+ public static LogicalPlan getLogicalPlan(PigServer pig) throws Exception {
+ java.lang.reflect.Method compileLp = pig.getClass()
+ .getDeclaredMethod("compileLp",
+ new Class[] { String.class });
+ compileLp.setAccessible(true);
+ return (LogicalPlan) compileLp.invoke(pig, new Object[] { null });
+ }
+
+ public static MROperPlan getMRPlan(PhysicalPlan pp, PigContext ctx) throws
Exception {
+ MapReduceLauncher launcher = new MapReduceLauncher();
+ java.lang.reflect.Method compile = launcher.getClass()
+ .getDeclaredMethod("compile",
+ new Class[] { PhysicalPlan.class, PigContext.class });
+ compile.setAccessible(true);
+ return (MROperPlan) compile.invoke(launcher, new Object[] { pp, ctx });
+ }
+
+ public static String getAlias(MapReduceOper mro) throws Exception {
+ ScriptState ss = ScriptState.get();
+ java.lang.reflect.Method getAlias = ss.getClass()
+ .getDeclaredMethod("getAlias",
+ new Class[] { MapReduceOper.class });
+ getAlias.setAccessible(true);
+ return (String)getAlias.invoke(ss, new Object[] { mro });
+ }
+
}
