Author: rding
Date: Fri Jan 22 22:21:36 2010
New Revision: 902295

URL: http://svn.apache.org/viewvc?rev=902295&view=rev
Log:
PIG-1090: Update sources to reflect recent changes in load-store interfaces

Modified:
    
hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/TestPigStorageSchema.java
    
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/ResourceSchema.java
    
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestResourceSchema.java

Modified: 
hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/TestPigStorageSchema.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/TestPigStorageSchema.java?rev=902295&r1=902294&r2=902295&view=diff
==============================================================================
--- 
hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/TestPigStorageSchema.java
 (original)
+++ 
hadoop/pig/branches/load-store-redesign/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/TestPigStorageSchema.java
 Fri Jan 22 22:21:36 2010
@@ -19,13 +19,19 @@
 
 package org.apache.pig.piggybank.test;
 
+import static org.junit.Assert.assertEquals;
+
 import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
 import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
+import org.apache.pig.ResourceSchema;
 import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.io.FileLocalizer;
 import org.apache.pig.impl.logicalLayer.LogicalOperator;
@@ -108,4 +114,80 @@
         assertTrue("explicit schema overrides metadata", 
Schema.equals(newSchema, newGenSchema, true, false));
         
     }
+    
+    @Test
+    public void testSchemaConversion() throws Exception {   
+ 
+        Util.createInputFile(cluster, "originput2", 
+                new String[] {"1", "2", "3", "2",
+                              "5", "5", "8", "8",
+                              "8", "9"});
+        
+        pig.registerQuery("A = LOAD 'originput2' using 
org.apache.pig.piggybank.storage.PigStorageSchema() as (f:int);");
+        pig.registerQuery("B = group A by f;");
+        Schema origSchema = pig.dumpSchema("B");
+        ResourceSchema rs1 = new ResourceSchema(origSchema);
+        pig.registerQuery("STORE B into 'bout' using 
org.apache.pig.piggybank.storage.PigStorageSchema();");
+        
+        pig.registerQuery("C = LOAD 'bout' using 
org.apache.pig.piggybank.storage.PigStorageSchema();");
+        Schema genSchema = pig.dumpSchema("C");
+        ResourceSchema rs2 = new ResourceSchema(genSchema);
+        assertTrue("generated schema equals original" , 
ResourceSchema.equals(rs1, rs2));
+        
+        pig.registerQuery("C1 = LOAD 'bout' as (a0:int, A: {t: (f:int) } );");
+        pig.registerQuery("D = foreach C1 generate a0, SUM(A);");
+
+        List<Tuple> expectedResults = Util.getTuplesFromConstantTupleStrings(
+                new String[] { 
+                        "(1,1L)",
+                        "(2,4L)",
+                        "(3,3L)",
+                        "(5,10L)",
+                        "(8,24L)",
+                        "(9,9L)"
+                });
+        
+        Iterator<Tuple> iter = pig.openIterator("D");
+        int counter = 0;
+        while (iter.hasNext()) {
+            assertEquals(expectedResults.get(counter++).toString(), 
iter.next().toString());      
+        }
+        
+        assertEquals(expectedResults.size(), counter);
+    }
+    
+    @Test
+    public void testSchemaConversion2() throws Exception {   
+ 
+        pig.registerQuery("A = LOAD 'originput' using 
org.apache.pig.piggybank.storage.PigStorageSchema(',') as (f1:chararray, 
f2:int);");
+        pig.registerQuery("B = group A by f1;");
+        Schema origSchema = pig.dumpSchema("B");
+        ResourceSchema rs1 = new ResourceSchema(origSchema);
+        pig.registerQuery("STORE B into 'bout' using 
org.apache.pig.piggybank.storage.PigStorageSchema();");
+        
+        pig.registerQuery("C = LOAD 'bout' using 
org.apache.pig.piggybank.storage.PigStorageSchema();");
+        Schema genSchema = pig.dumpSchema("C");
+        ResourceSchema rs2 = new ResourceSchema(genSchema);
+        assertTrue("generated schema equals original" , 
ResourceSchema.equals(rs1, rs2));
+        
+        pig.registerQuery("C1 = LOAD 'bout' as (a0:chararray, A: {t: 
(f1:chararray, f2:int) } );");
+        pig.registerQuery("D = foreach C1 generate a0, SUM(A.f2);");
+
+        List<Tuple> expectedResults = Util.getTuplesFromConstantTupleStrings(
+                new String[] { 
+                        "('A',23L)",
+                        "('B',7L)",
+                        "('C',11L)",
+                        "('D',10L)"
+                });
+        
+        Iterator<Tuple> iter = pig.openIterator("D");
+        int counter = 0;
+        while (iter.hasNext()) {
+            assertEquals(expectedResults.get(counter++).toString(), 
iter.next().toString());      
+        }
+        
+        assertEquals(expectedResults.size(), counter);
+    }
+ 
 }

Modified: 
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/ResourceSchema.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/ResourceSchema.java?rev=902295&r1=902294&r2=902295&view=diff
==============================================================================
--- 
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/ResourceSchema.java 
(original)
+++ 
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/ResourceSchema.java 
Fri Jan 22 22:21:36 2010
@@ -22,6 +22,8 @@
 import java.util.Arrays;
 import java.util.List;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.pig.data.DataType;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
@@ -29,6 +31,9 @@
 public class ResourceSchema implements Serializable {
 
     private static final long serialVersionUID = 1L;
+    
+    private static Log log = LogFactory.getLog(ResourceSchema.class);
+    
     /* Array Getters intentionally return mutable arrays instead of copies,
      * to simplify updates without unnecessary copying.
      * Setters make a copy of the arrays in order to prevent an array
@@ -66,16 +71,18 @@
             type = fieldSchema.type;
             name = fieldSchema.alias;
             description = "autogenerated from Pig Field Schema";
-            if (type == DataType.BAG && fieldSchema.schema != null) { // allow 
partial schema
-                List<FieldSchema> slst = fieldSchema.schema.getFields();
-                if (slst.size() != 1 || slst.get(0).type != DataType.TUPLE) {
-                    throw new IllegalArgumentException("Invalid Pig schema: " +
-                               "bag schema must have tuple as its field.");
-                }
+            Schema inner = fieldSchema.schema;
+            if (type == DataType.BAG && fieldSchema.schema != null
+                    && !fieldSchema.schema.isTwoLevelAccessRequired()) { 
+                log.info("Insert two-level access to Resource Schema");
+                FieldSchema fs = new FieldSchema("t", fieldSchema.schema);
+                inner = new Schema(fs);                
             }
-            // XXX allow partial schema 
-            if (type == DataType.BAG || type == DataType.TUPLE) {
-                schema = new ResourceSchema(fieldSchema.schema);
+            
+            // allow partial schema 
+            if ((type == DataType.BAG || type == DataType.TUPLE)
+                    && inner != null) {
+                schema = new ResourceSchema(inner);
             } else {
                 schema = null;
             }

Modified: 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestResourceSchema.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestResourceSchema.java?rev=902295&r1=902294&r2=902295&view=diff
==============================================================================
--- 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestResourceSchema.java
 (original)
+++ 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestResourceSchema.java
 Fri Jan 22 22:21:36 2010
@@ -194,9 +194,9 @@
     }
     
     /**
-     * Test invalid Pig Schema: multiple fields for a bag
+     * Test one-level Pig Schema: multiple fields for a bag
      */
-    @Test(expected=IllegalArgumentException.class) 
+    @Test
     public void testResourceSchemaWithInvalidPigSchema() 
     throws FrontendException {
         String [] aliases ={"f1", "f2"};
@@ -206,13 +206,14 @@
         Schema.FieldSchema fld0 = 
             new Schema.FieldSchema("f0", level0, DataType.BAG);
         Schema level1 = new Schema(fld0);
-        new ResourceSchema(level1);
+        Schema genSchema = Schema.getPigSchema(new ResourceSchema(level1));
+        assertTrue(CheckTwoLevelAccess(genSchema));
     }
     
     /**
-     * Test invalid Pig Schema: bag without tuple field
+     * Test one-level Pig Schema: bag without tuple field
      */
-    @Test(expected=IllegalArgumentException.class) 
+    @Test
     public void testResourceSchemaWithInvalidPigSchema2() 
     throws FrontendException {
         String [] aliases ={"f1"};
@@ -222,6 +223,7 @@
         Schema.FieldSchema fld0 = 
             new Schema.FieldSchema("f0", level0, DataType.BAG);
         Schema level1 = new Schema(fld0);
-        new ResourceSchema(level1);
+        Schema genSchema = Schema.getPigSchema(new ResourceSchema(level1));
+        assertTrue(CheckTwoLevelAccess(genSchema));
     }
 }


Reply via email to