Modified: 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPruneColumn.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPruneColumn.java?rev=908262&r1=908261&r2=908262&view=diff
==============================================================================
--- 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPruneColumn.java
 (original)
+++ 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPruneColumn.java
 Tue Feb  9 22:40:13 2010
@@ -33,8 +33,11 @@
 import org.apache.log4j.SimpleLayout;
 import org.apache.pig.ExecType;
 import org.apache.pig.FilterFunc;
+import org.apache.pig.LoadPushDown;
 import org.apache.pig.PigServer;
+import org.apache.pig.builtin.PigStorage;
 import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.logicalLayer.optimizer.PruneColumns;
 import org.junit.Before;
 import org.junit.Test;
@@ -55,8 +58,24 @@
     File tmpFile9;
     File tmpFile10;
     File logFile;
-    Logger logger;
 
+    static public class PigStorageWithTrace extends PigStorage {
+
+        /**
+         * @param delimiter
+         */
+        public PigStorageWithTrace() {
+            super();
+        }
+        @Override
+        public RequiredFieldResponse pushProjection(RequiredFieldList 
requiredFieldList) throws FrontendException {
+            RequiredFieldResponse response = 
super.pushProjection(requiredFieldList);
+            Logger logger = Logger.getLogger(this.getClass());
+            logger.info(requiredFieldList);
+            return response;
+        }
+
+    }
     private static final String simpleEchoStreamingCommand;
     static {
         if (System.getProperty("os.name").toUpperCase().startsWith("WINDOWS"))
@@ -76,7 +95,7 @@
     @Before
     @Override
     public void setUp() throws Exception{
-        logger = Logger.getLogger(PruneColumns.class);
+        Logger logger = Logger.getLogger(PruneColumns.class);
         logger.removeAllAppenders();
         logger.setLevel(Level.INFO);
         SimpleLayout layout = new SimpleLayout();
@@ -84,6 +103,10 @@
         FileAppender appender = new FileAppender(layout, logFile.toString(), 
false, false, 0);
         logger.addAppender(appender);
         
+        Logger pigStorageWithTraceLogger = 
Logger.getLogger(PigStorageWithTrace.class);
+        pigStorageWithTraceLogger.setLevel(Level.INFO);
+        pigStorageWithTraceLogger.addAppender(appender);
+        
         pigServer = new PigServer("local");
         //pigServer = new PigServer(ExecType.MAPREDUCE, 
cluster.getProperties());
         tmpFile1 = File.createTempFile("prune", "txt");
@@ -1767,4 +1790,59 @@
 
         assertTrue(emptyLogFileMessage());
     }
+    
+    // See PIG-1184
+    @Test
+    public void testForEachFlatten() throws Exception {
+        File inputFile = Util.createInputFile("table_testForEachFlatten", "", 
new String[]{"oiue\tM\t{(3),(4)}\t{(toronto),(montreal)}"});
+        
+        pigServer.registerQuery("A = load '"+inputFile.toString()+"' as 
(a0:chararray, a1:chararray, a2:bag{t:tuple(id:chararray)}, 
a3:bag{t:tuple(loc:chararray)});");
+        pigServer.registerQuery("B = foreach A generate a0, a1, flatten(a2), 
flatten(a3), 10;");
+        pigServer.registerQuery("C = foreach B generate a0, $4;");
+        Iterator<Tuple> iter = pigServer.openIterator("C");
+
+        assertTrue(iter.hasNext());
+        Tuple t = iter.next();
+        assertTrue(t.toString().equals("(oiue,10)"));
+
+        assertTrue(iter.hasNext());
+        t = iter.next();
+        assertTrue(t.toString().equals("(oiue,10)"));
+
+        assertTrue(iter.hasNext());
+        t = iter.next();
+        assertTrue(t.toString().equals("(oiue,10)"));
+
+        assertTrue(iter.hasNext());
+        t = iter.next();
+        assertTrue(t.toString().equals("(oiue,10)"));
+
+        assertFalse(iter.hasNext());
+
+        assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $1",
+                "No map keys pruned for A"}));
+    }
+    
+    // See PIG-1210
+    @Test
+    public void testFieldsToReadDuplicatedEntry() throws Exception {
+        pigServer.registerQuery("A = load '"+ 
Util.generateURI(tmpFile1.toString(), pigServer.getPigContext()) + "' using 
"+PigStorageWithTrace.class.getName()
+                +" AS (a0, a1, a2);");
+        pigServer.registerQuery("B = foreach A generate a0+a0, a1, a2;");
+        Iterator<Tuple> iter = pigServer.openIterator("B");
+
+        assertTrue(iter.hasNext());
+        Tuple t = iter.next();
+        assertTrue(t.toString().equals("(2.0,2,3)"));
+        
+        assertTrue(iter.hasNext());
+        t = iter.next();
+        assertTrue(t.toString().equals("(4.0,5,2)"));
+
+        assertFalse(iter.hasNext());
+
+        assertTrue(checkLogFileMessage(new String[]{"No column pruned for A",
+            "No map keys pruned for A", "[0,1,2]"}));
+    }
+
 }

Modified: 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSchema.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSchema.java?rev=908262&r1=908261&r2=908262&view=diff
==============================================================================
--- 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSchema.java
 (original)
+++ 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSchema.java
 Tue Feb  9 22:40:13 2010
@@ -36,16 +36,22 @@
 
 package org.apache.pig.test;
 
-import java.util.* ;
-
-import org.apache.pig.data.* ;
-import org.apache.pig.impl.logicalLayer.schema.* ;
-import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-
-import org.junit.* ;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
 
 import junit.framework.Assert;
-import junit.framework.TestCase ;
+import junit.framework.TestCase;
+
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.logicalLayer.schema.SchemaMergeException;
+import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+import org.junit.Test;
 
 public class TestSchema extends TestCase {
     
@@ -634,4 +640,20 @@
                Assert.assertTrue(Schema.FieldSchema.castable(castFieldSchema, 
inputFieldSchema));
        }
     }
+    
+    public void testSchemaSerialization() throws IOException {
+        MiniCluster cluster = MiniCluster.buildCluster();
+        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, 
cluster.getProperties());
+        String inputFileName = "testSchemaSerialization-input.txt";
+        String[] inputData = new String[] { "foo\t1", "hello\t2" };
+        Util.createInputFile(cluster, inputFileName, inputData);
+        String script = "a = load '"+ inputFileName +"' as (f1:chararray, 
f2:int);" +
+                       " b = group a all; c = foreach b generate 
org.apache.pig.test.InputSchemaUDF(a);";
+        Util.registerMultiLineQuery(pigServer, script);
+        Iterator<Tuple> it = pigServer.openIterator("c");
+        while(it.hasNext()) {
+            Tuple t = it.next();
+            assertEquals("{a: {f1: chararray,f2: int}}", t.get(0));
+        }
+    }
 }

Modified: 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStreamingLocal.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStreamingLocal.java?rev=908262&r1=908261&r2=908262&view=diff
==============================================================================
--- 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStreamingLocal.java
 (original)
+++ 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStreamingLocal.java
 Tue Feb  9 22:40:13 2010
@@ -18,6 +18,8 @@
 package org.apache.pig.test;
 
 import java.io.File;
+import java.util.ArrayList;
+import java.util.Iterator;
 
 import junit.framework.TestCase;
 
@@ -286,6 +288,39 @@
             Util.checkQueryOutputs(pigServer.openIterator("OP"), 
expectedResults);
         }
     }
+    
+    @Test
+    public void testJoinTwoStreamingRelations() 
+    throws Exception {
+        ArrayList<String> list = new ArrayList<String>();
+        for (int i=0; i<10000; i++) {
+            list.add("A," + i);
+        }
+        File input = Util.createInputFile("tmp", "", list.toArray(new 
String[0]));
+        
+        // Expected results
+        Tuple expected = DefaultTupleFactory.getInstance().newTuple(4);
+        expected.set(0, "A");
+        expected.set(1, 0);
+        expected.set(2, "A");
+        expected.set(3, 0);        
+ 
+        pigServer.registerQuery("A = load 'file:" + 
Util.encodeEscape(input.toString()) + "' using " + 
+                    PigStorage.class.getName() + "(',') as (a0, a1);");
+        pigServer.registerQuery("B = stream A through `head -1` as (a0, a1);");
+        pigServer.registerQuery("C = load 'file:" + 
Util.encodeEscape(input.toString()) + "' using " + 
+                PigStorage.class.getName() + "(',') as (a0, a1);");
+        pigServer.registerQuery("D = stream C through `head -1` as (a0, a1);");
+        pigServer.registerQuery("E = join B by a0, D by a0;");
+        
+        Iterator<Tuple> iter = pigServer.openIterator("E");
+        int count = 0;
+        while (iter.hasNext()) {
+            Assert.assertEquals(expected.toString(), iter.next().toString());
+            count++;
+        }
+        Assert.assertTrue(count == 1);
+    }
 
     @Test
     public void testLocalNegativeLoadStoreOptimization() throws Exception {

Modified: 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld?rev=908262&r1=908261&r2=908262&view=diff
==============================================================================
--- 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld
 (original)
+++ 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld
 Tue Feb  9 22:40:13 2010
@@ -26,7 +26,7 @@
     |   |
     |   
|---Load(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage)
 - -152
     |
-    |---MapReduce(20,TestMRCompiler$WeirdComparator,COUNT,SUM) - -145:
+    |---MapReduce(20,SUM,COUNT,TestMRCompiler$WeirdComparator) - -145:
         |   
Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) 
- -151
         |   |
         |   |---New For Each(false,false)[tuple] - -2197807331204639125
@@ -88,4 +88,4 @@
                 |       |   |
                 |       |   Constant(true) - --6860857982727545176
                 |       |
-                |       |---Load(DummyFil:DummyLdr) - -737965411848544085
\ No newline at end of file
+                |       |---Load(DummyFil:DummyLdr) - -737965411848544085

Modified: 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/GenPhyOp.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/GenPhyOp.java?rev=908262&r1=908261&r2=908262&view=diff
==============================================================================
--- 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/GenPhyOp.java
 (original)
+++ 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/GenPhyOp.java
 Tue Feb  9 22:40:13 2010
@@ -762,7 +762,6 @@
 
     public static POStore topStoreOp() {
         POStore ret = new POStore(new OperatorKey("", r.nextLong()));
-        ret.setSFile(new FileSpec("DummyFil", new FuncSpec("DummyLdr")));
         return ret;
     }
 


Reply via email to