Author: gates Date: Tue Apr 22 15:58:28 2008 New Revision: 650688 URL: http://svn.apache.org/viewvc?rev=650688&view=rev Log: PIG-207 Fix illustrate command to work in mapreduce mode.
Modified: incubator/pig/trunk/CHANGES.txt incubator/pig/trunk/src/org/apache/pig/pen/DisplayExamples.java incubator/pig/trunk/src/org/apache/pig/pen/ExGen.java incubator/pig/trunk/test/org/apache/pig/test/TestExGenCogroup.java incubator/pig/trunk/test/org/apache/pig/test/TestExGenEval.java Modified: incubator/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=650688&r1=650687&r2=650688&view=diff ============================================================================== --- incubator/pig/trunk/CHANGES.txt (original) +++ incubator/pig/trunk/CHANGES.txt Tue Apr 22 15:58:28 2008 @@ -246,3 +246,7 @@ PIG-216 Fix streaming to work with commands that use unix pipes (acmurthy via gates). + + PIG-207 Fix illustrate command to work in mapreduce mode (shubhamc via + gates). + Modified: incubator/pig/trunk/src/org/apache/pig/pen/DisplayExamples.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/pen/DisplayExamples.java?rev=650688&r1=650687&r2=650688&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/pen/DisplayExamples.java (original) +++ incubator/pig/trunk/src/org/apache/pig/pen/DisplayExamples.java Tue Apr 22 15:58:28 2008 @@ -38,7 +38,7 @@ public class DisplayExamples { public static StringBuffer Result = new StringBuffer(); - public static final int MAX_DATAATOM_LENGTH = 10; + public static final int MAX_DATAATOM_LENGTH = 15; public static String PrintTabular(LogicalPlan lp, Map<LogicalOperator, DataBag> exampleData) { StringBuffer output = new StringBuffer(); @@ -96,13 +96,13 @@ } //Display the schema first - output.append(AddSpaces(total + 3*cols + 9, false) + "\n"); + output.append(AddSpaces(total + 3*(cols +1) + aliasLength + 1, false) + "\n"); output.append("| " + op.getAlias() + AddSpaces(4, true) + " | "); for(int i = 0; i < cols; ++i) { String field = fields.get(i).toString(); output.append(field + AddSpaces(maxColSizes[i] - field.length(), true) + " | "); } - output.append("\n" + AddSpaces(total + 3*cols + 9, false) + "\n"); + output.append("\n" + AddSpaces(total + 3*(cols +1) + aliasLength + 1, false) + "\n"); //now start displaying the data for(int i = 0; i < rows; ++i) { output.append("| " + AddSpaces(aliasLength, true) + " | "); @@ -113,7 +113,7 @@ output.append("\n"); } //now display the finish line - output.append(AddSpaces(total + 3*cols + 9, false) + "\n"); + output.append(AddSpaces(total + 3*(cols +1) + aliasLength + 1, false) + "\n"); } static String[][] MakeArray(LogicalOperator op, DataBag bag) { @@ -144,7 +144,7 @@ static String ShortenField(DataAtom da) { int length = da.toString().length(); if(length > MAX_DATAATOM_LENGTH) { - return new String(da.toString().substring(0, 3) + " ... " + da.toString().substring(length - 4, length - 1)); + return new String(da.toString().substring(0, 4) + " ... " + da.toString().substring(length - 4, length - 1)); } return da.toString(); } @@ -159,7 +159,7 @@ while(it.hasNext()) { Tuple t = it.next(); if(!it.hasNext()) { - str.append(", ... " + ShortenField(t)); + str.append(", ..., " + ShortenField(t)); } } } else { @@ -182,7 +182,7 @@ if(noFields > 3) { Datum d = t.getField(0); - str.append(ShortenField(d) + ", ..."); + str.append(ShortenField(d) + ", ..., "); d = t.getField(noFields - 1); str.append(ShortenField(d)); Modified: incubator/pig/trunk/src/org/apache/pig/pen/ExGen.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/pen/ExGen.java?rev=650688&r1=650687&r2=650688&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/pen/ExGen.java (original) +++ incubator/pig/trunk/src/org/apache/pig/pen/ExGen.java Tue Apr 22 15:58:28 2008 @@ -24,6 +24,7 @@ import java.util.LinkedList; import java.util.Map; +import org.apache.pig.PigServer.ExecType; import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.backend.executionengine.ExecPhysicalOperator; import org.apache.pig.backend.executionengine.ExecPhysicalPlan; @@ -45,14 +46,16 @@ static Map<LOLoad, DataBag> GlobalBaseData = new HashMap<LOLoad, DataBag>(); - public static Map<LogicalOperator, DataBag> GenerateExamples(LogicalPlan plan, PigContext pigContext) throws IOException { + public static Map<LogicalOperator, DataBag> GenerateExamples(LogicalPlan plan, PigContext hadoopPigContext) throws IOException { long time = System.currentTimeMillis(); String Result; + PigContext pigContext = new PigContext(ExecType.LOCAL, hadoopPigContext.getProperties()); //compile the logical plan to get the physical plan once and for all ExecPhysicalPlan PhyPlan = null; try { + pigContext.connect(); PhyPlan = pigContext.getExecutionEngine().compile(plan, null); } catch (ExecException e1) { // TODO Auto-generated catch block @@ -62,7 +65,7 @@ Map<OperatorKey, ExecPhysicalOperator> physicalOpTable = PhyPlan.getOpTable(); // Acquire initial base data by sampling from input relations (this is idempotent) - FetchBaseData.ReadBaseData(plan.getRootOperator(), GlobalBaseData, SAMPLE_SIZE, pigContext); + FetchBaseData.ReadBaseData(plan.getRootOperator(), GlobalBaseData, SAMPLE_SIZE, hadoopPigContext); /////// PASS 1: push data sample through query plan Modified: incubator/pig/trunk/test/org/apache/pig/test/TestExGenCogroup.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestExGenCogroup.java?rev=650688&r1=650687&r2=650688&view=diff ============================================================================== --- incubator/pig/trunk/test/org/apache/pig/test/TestExGenCogroup.java (original) +++ incubator/pig/trunk/test/org/apache/pig/test/TestExGenCogroup.java Tue Apr 22 15:58:28 2008 @@ -5,6 +5,8 @@ import java.util.Random; import org.apache.pig.PigServer; +import org.apache.pig.PigServer.ExecType; +import org.apache.pig.impl.io.FileLocalizer; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -12,21 +14,29 @@ import junit.framework.TestCase; public class TestExGenCogroup extends TestCase{ - File A, B; + String A, B; private int MAX = 10; - String initString = "local"; + String initString = "mapreduce"; PigServer pig; + MiniCluster cluster = MiniCluster.buildCluster(); + @Override @Before protected void setUp() throws Exception{ + File fileA, fileB; + pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties()); System.out.println("Generating test data..."); - A = File.createTempFile("dataA", ".dat"); - B = File.createTempFile("dataB", ".dat"); + fileA = File.createTempFile("dataA", ".dat"); + fileB = File.createTempFile("dataB", ".dat"); + + writeData(fileA); + writeData(fileB); - writeData(A); - writeData(B); + A = "'" + FileLocalizer.hadoopify(fileA.toString(), pig.getPigContext()) + "'"; + B = "'" + FileLocalizer.hadoopify(fileB.toString(), pig.getPigContext()) + "'"; + System.out.println("A : " + A + "\n" + "B : " + B); System.out.println("Test data created."); } @@ -39,39 +49,39 @@ for(int i = 0; i < MAX; i++) dat.write((rand.nextInt(10) + "\t" + rand.nextInt(10) + "\n").getBytes()); + dat.close(); } @Override @After protected void tearDown() throws Exception { - A.delete(); - B.delete(); + } @Test public void testCogroupMultipleCols() throws Exception { - pig = new PigServer(initString); - pig.registerQuery("A = load '" + A.toString() + "' as (x, y);"); - pig.registerQuery("B = load '" + B.toString() + "' as (x, y);"); + //pig = new PigServer(initString); + pig.registerQuery("A = load " + A + " as (x, y);"); + pig.registerQuery("B = load " + B + " as (x, y);"); pig.registerQuery("C = cogroup A by (x, y), B by (x, y);"); pig.showExamples("C"); } @Test public void testCogroup() throws Exception { - pig = new PigServer(initString); - pig.registerQuery("A = load '" + A.toString() + "' as (x, y);"); - pig.registerQuery("B = load '" + B.toString() + "' as (x, y);"); + //pig = new PigServer(initString); + pig.registerQuery("A = load " + A + " as (x, y);"); + pig.registerQuery("B = load " + B + " as (x, y);"); pig.registerQuery("C = cogroup A by x, B by x;"); pig.showExamples("C"); } @Test public void testGroup() throws Exception { - pig = new PigServer(initString); - pig.registerQuery("A = load '" + A.toString() + "' as (x, y);"); + //pig = new PigServer(initString); + pig.registerQuery("A = load " + A.toString() + " as (x, y);"); pig.registerQuery("B = group A by x;"); pig.showExamples("B"); @@ -79,9 +89,9 @@ @Test public void testComplexGroup() throws Exception { - pig = new PigServer(initString); - pig.registerQuery("A = load '" + A.toString() + "' as (x, y);"); - pig.registerQuery("B = load '" + B.toString() + "' as (x, y);"); + //pig = new PigServer(initString); + pig.registerQuery("A = load " + A.toString() + " as (x, y);"); + pig.registerQuery("B = load " + B.toString() + " as (x, y);"); pig.registerQuery("C = cogroup A by x, B by x;"); pig.registerQuery("D = cogroup A by y, B by y;"); pig.registerQuery("E = cogroup C by $0, D by $0;"); Modified: incubator/pig/trunk/test/org/apache/pig/test/TestExGenEval.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestExGenEval.java?rev=650688&r1=650687&r2=650688&view=diff ============================================================================== --- incubator/pig/trunk/test/org/apache/pig/test/TestExGenEval.java (original) +++ incubator/pig/trunk/test/org/apache/pig/test/TestExGenEval.java Tue Apr 22 15:58:28 2008 @@ -2,9 +2,14 @@ import java.io.File; import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; import java.util.Random; import org.apache.pig.PigServer; +import org.apache.pig.PigServer.ExecType; +import org.apache.pig.impl.PigContext; +import org.apache.pig.impl.io.FileLocalizer; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -13,26 +18,45 @@ public class TestExGenEval extends TestCase { - File A, B, C, D; + String A, B, C, D; private int MAX = 10; - String initString = "local"; + String initString = "mapreduce"; PigServer pig; + PigContext pigContext; + + MiniCluster cluster = MiniCluster.buildCluster(); @Override @Before protected void setUp() throws Exception{ System.out.println("Generating test data..."); - A = File.createTempFile("dataA", ".dat"); - B = File.createTempFile("dataB", ".dat"); - C = File.createTempFile("dataC", ".dat"); - D = File.createTempFile("dataD", ".dat"); - - writeData(A); - writeData(B); - writeData(C); - writeData(D); + File fileA, fileB, fileC, fileD; + + pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties()); + pigContext = pig.getPigContext(); + fileA = File.createTempFile("dataA", ".dat"); + fileB = File.createTempFile("dataB", ".dat"); + fileC = File.createTempFile("dataC", ".dat"); + fileD = File.createTempFile("dataD", ".dat"); + + + writeData(fileA); + writeData(fileB); + writeData(fileC); + writeData(fileD); + + + A = "'" + FileLocalizer.hadoopify(fileA.toString(), pig.getPigContext()) + "'"; + B = "'" + FileLocalizer.hadoopify(fileB.toString(), pig.getPigContext()) + "'"; + C = "'" + FileLocalizer.hadoopify(fileC.toString(), pig.getPigContext()) + "'"; + D = "'" + FileLocalizer.hadoopify(fileD.toString(), pig.getPigContext()) + "'"; + System.out.println("Test data created."); + fileA.delete(); + fileB.delete(); + fileC.delete(); + fileD.delete(); } @@ -50,17 +74,14 @@ @Override @After protected void tearDown() throws Exception { - A.delete(); - B.delete(); - C.delete(); - D.delete(); + } @Test public void testForeach() throws Exception { - pig = new PigServer(initString); + //pig = new PigServer(initString); System.out.println("Testing Foreach statement..."); - pig.registerQuery("A = load '" + A.toString() + "' as (x, y);"); + pig.registerQuery("A = load " + A + " as (x, y);"); pig.registerQuery("B = foreach A generate x+y as sum;"); pig.showExamples("B"); assertEquals(1, 1); @@ -68,8 +89,8 @@ @Test public void testFilter() throws Exception { - pig = new PigServer(initString); - pig.registerQuery("A = load '" + A.toString() + "' as (x, y);"); + //pig = new PigServer(initString); + pig.registerQuery("A = load " + A + " as (x, y);"); pig.registerQuery("B = filter A by x < 10.0;"); pig.showExamples("B"); assertEquals(1, 1); @@ -77,11 +98,11 @@ @Test public void testFlatten() throws Exception { - pig = new PigServer(initString); - pig.registerQuery("A1 = load '" + A.toString() + "' as (x, y);"); - pig.registerQuery("B1 = load '" + B.toString() + "' as (x, y);"); - pig.registerQuery("C1 = load '" + C.toString() + "' as (x, y);"); - pig.registerQuery("D1 = load '" + D.toString() + "' as (x, y);"); + //pig = new PigServer(initString); + pig.registerQuery("A1 = load " + A + " as (x, y);"); + pig.registerQuery("B1 = load " + B + " as (x, y);"); + pig.registerQuery("C1 = load " + C + " as (x, y);"); + pig.registerQuery("D1 = load " + D + " as (x, y);"); pig.registerQuery("E = join A1 by x, B1 by x;"); pig.registerQuery("F = join C1 by x, D1 by x;"); pig.registerQuery("G = join E by $0, F by $0;");