Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPruneColumn.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPruneColumn.java?rev=908262&r1=908261&r2=908262&view=diff ============================================================================== --- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPruneColumn.java (original) +++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPruneColumn.java Tue Feb 9 22:40:13 2010 @@ -33,8 +33,11 @@ import org.apache.log4j.SimpleLayout; import org.apache.pig.ExecType; import org.apache.pig.FilterFunc; +import org.apache.pig.LoadPushDown; import org.apache.pig.PigServer; +import org.apache.pig.builtin.PigStorage; import org.apache.pig.data.Tuple; +import org.apache.pig.impl.logicalLayer.FrontendException; import org.apache.pig.impl.logicalLayer.optimizer.PruneColumns; import org.junit.Before; import org.junit.Test; @@ -55,8 +58,24 @@ File tmpFile9; File tmpFile10; File logFile; - Logger logger; + static public class PigStorageWithTrace extends PigStorage { + + /** + * @param delimiter + */ + public PigStorageWithTrace() { + super(); + } + @Override + public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldList) throws FrontendException { + RequiredFieldResponse response = super.pushProjection(requiredFieldList); + Logger logger = Logger.getLogger(this.getClass()); + logger.info(requiredFieldList); + return response; + } + + } private static final String simpleEchoStreamingCommand; static { if (System.getProperty("os.name").toUpperCase().startsWith("WINDOWS")) @@ -76,7 +95,7 @@ @Before @Override public void setUp() throws Exception{ - logger = Logger.getLogger(PruneColumns.class); + Logger logger = Logger.getLogger(PruneColumns.class); logger.removeAllAppenders(); logger.setLevel(Level.INFO); SimpleLayout layout = new SimpleLayout(); @@ -84,6 +103,10 @@ FileAppender appender = new FileAppender(layout, logFile.toString(), false, false, 0); logger.addAppender(appender); + Logger pigStorageWithTraceLogger = Logger.getLogger(PigStorageWithTrace.class); + pigStorageWithTraceLogger.setLevel(Level.INFO); + pigStorageWithTraceLogger.addAppender(appender); + pigServer = new PigServer("local"); //pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties()); tmpFile1 = File.createTempFile("prune", "txt"); @@ -1767,4 +1790,59 @@ assertTrue(emptyLogFileMessage()); } + + // See PIG-1184 + @Test + public void testForEachFlatten() throws Exception { + File inputFile = Util.createInputFile("table_testForEachFlatten", "", new String[]{"oiue\tM\t{(3),(4)}\t{(toronto),(montreal)}"}); + + pigServer.registerQuery("A = load '"+inputFile.toString()+"' as (a0:chararray, a1:chararray, a2:bag{t:tuple(id:chararray)}, a3:bag{t:tuple(loc:chararray)});"); + pigServer.registerQuery("B = foreach A generate a0, a1, flatten(a2), flatten(a3), 10;"); + pigServer.registerQuery("C = foreach B generate a0, $4;"); + Iterator<Tuple> iter = pigServer.openIterator("C"); + + assertTrue(iter.hasNext()); + Tuple t = iter.next(); + assertTrue(t.toString().equals("(oiue,10)")); + + assertTrue(iter.hasNext()); + t = iter.next(); + assertTrue(t.toString().equals("(oiue,10)")); + + assertTrue(iter.hasNext()); + t = iter.next(); + assertTrue(t.toString().equals("(oiue,10)")); + + assertTrue(iter.hasNext()); + t = iter.next(); + assertTrue(t.toString().equals("(oiue,10)")); + + assertFalse(iter.hasNext()); + + assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $1", + "No map keys pruned for A"})); + } + + // See PIG-1210 + @Test + public void testFieldsToReadDuplicatedEntry() throws Exception { + pigServer.registerQuery("A = load '"+ Util.generateURI(tmpFile1.toString(), pigServer.getPigContext()) + "' using "+PigStorageWithTrace.class.getName() + +" AS (a0, a1, a2);"); + pigServer.registerQuery("B = foreach A generate a0+a0, a1, a2;"); + Iterator<Tuple> iter = pigServer.openIterator("B"); + + assertTrue(iter.hasNext()); + Tuple t = iter.next(); + assertTrue(t.toString().equals("(2.0,2,3)")); + + assertTrue(iter.hasNext()); + t = iter.next(); + assertTrue(t.toString().equals("(4.0,5,2)")); + + assertFalse(iter.hasNext()); + + assertTrue(checkLogFileMessage(new String[]{"No column pruned for A", + "No map keys pruned for A", "[0,1,2]"})); + } + }
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSchema.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSchema.java?rev=908262&r1=908261&r2=908262&view=diff ============================================================================== --- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSchema.java (original) +++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSchema.java Tue Feb 9 22:40:13 2010 @@ -36,16 +36,22 @@ package org.apache.pig.test; -import java.util.* ; - -import org.apache.pig.data.* ; -import org.apache.pig.impl.logicalLayer.schema.* ; -import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema; - -import org.junit.* ; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; import junit.framework.Assert; -import junit.framework.TestCase ; +import junit.framework.TestCase; + +import org.apache.pig.ExecType; +import org.apache.pig.PigServer; +import org.apache.pig.data.DataType; +import org.apache.pig.data.Tuple; +import org.apache.pig.impl.logicalLayer.schema.Schema; +import org.apache.pig.impl.logicalLayer.schema.SchemaMergeException; +import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema; +import org.junit.Test; public class TestSchema extends TestCase { @@ -634,4 +640,20 @@ Assert.assertTrue(Schema.FieldSchema.castable(castFieldSchema, inputFieldSchema)); } } + + public void testSchemaSerialization() throws IOException { + MiniCluster cluster = MiniCluster.buildCluster(); + PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties()); + String inputFileName = "testSchemaSerialization-input.txt"; + String[] inputData = new String[] { "foo\t1", "hello\t2" }; + Util.createInputFile(cluster, inputFileName, inputData); + String script = "a = load '"+ inputFileName +"' as (f1:chararray, f2:int);" + + " b = group a all; c = foreach b generate org.apache.pig.test.InputSchemaUDF(a);"; + Util.registerMultiLineQuery(pigServer, script); + Iterator<Tuple> it = pigServer.openIterator("c"); + while(it.hasNext()) { + Tuple t = it.next(); + assertEquals("{a: {f1: chararray,f2: int}}", t.get(0)); + } + } } Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStreamingLocal.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStreamingLocal.java?rev=908262&r1=908261&r2=908262&view=diff ============================================================================== --- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStreamingLocal.java (original) +++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStreamingLocal.java Tue Feb 9 22:40:13 2010 @@ -18,6 +18,8 @@ package org.apache.pig.test; import java.io.File; +import java.util.ArrayList; +import java.util.Iterator; import junit.framework.TestCase; @@ -286,6 +288,39 @@ Util.checkQueryOutputs(pigServer.openIterator("OP"), expectedResults); } } + + @Test + public void testJoinTwoStreamingRelations() + throws Exception { + ArrayList<String> list = new ArrayList<String>(); + for (int i=0; i<10000; i++) { + list.add("A," + i); + } + File input = Util.createInputFile("tmp", "", list.toArray(new String[0])); + + // Expected results + Tuple expected = DefaultTupleFactory.getInstance().newTuple(4); + expected.set(0, "A"); + expected.set(1, 0); + expected.set(2, "A"); + expected.set(3, 0); + + pigServer.registerQuery("A = load 'file:" + Util.encodeEscape(input.toString()) + "' using " + + PigStorage.class.getName() + "(',') as (a0, a1);"); + pigServer.registerQuery("B = stream A through `head -1` as (a0, a1);"); + pigServer.registerQuery("C = load 'file:" + Util.encodeEscape(input.toString()) + "' using " + + PigStorage.class.getName() + "(',') as (a0, a1);"); + pigServer.registerQuery("D = stream C through `head -1` as (a0, a1);"); + pigServer.registerQuery("E = join B by a0, D by a0;"); + + Iterator<Tuple> iter = pigServer.openIterator("E"); + int count = 0; + while (iter.hasNext()) { + Assert.assertEquals(expected.toString(), iter.next().toString()); + count++; + } + Assert.assertTrue(count == 1); + } @Test public void testLocalNegativeLoadStoreOptimization() throws Exception { Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld?rev=908262&r1=908261&r2=908262&view=diff ============================================================================== --- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld (original) +++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld Tue Feb 9 22:40:13 2010 @@ -26,7 +26,7 @@ | | | |---Load(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -152 | - |---MapReduce(20,TestMRCompiler$WeirdComparator,COUNT,SUM) - -145: + |---MapReduce(20,SUM,COUNT,TestMRCompiler$WeirdComparator) - -145: | Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -151 | | | |---New For Each(false,false)[tuple] - -2197807331204639125 @@ -88,4 +88,4 @@ | | | | | Constant(true) - --6860857982727545176 | | - | |---Load(DummyFil:DummyLdr) - -737965411848544085 \ No newline at end of file + | |---Load(DummyFil:DummyLdr) - -737965411848544085 Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/GenPhyOp.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/GenPhyOp.java?rev=908262&r1=908261&r2=908262&view=diff ============================================================================== --- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/GenPhyOp.java (original) +++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/GenPhyOp.java Tue Feb 9 22:40:13 2010 @@ -762,7 +762,6 @@ public static POStore topStoreOp() { POStore ret = new POStore(new OperatorKey("", r.nextLong())); - ret.setSFile(new FileSpec("DummyFil", new FuncSpec("DummyLdr"))); return ret; }
