Author: olga
Date: Mon Sep 22 12:33:26 2008
New Revision: 697958

URL: http://svn.apache.org/viewvc?rev=697958&view=rev
Log:
PIG-424: nested foreach with flatten and agg gives an error

Modified:
    incubator/pig/branches/types/CHANGES.txt
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
    
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java

Modified: incubator/pig/branches/types/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/CHANGES.txt?rev=697958&r1=697957&r2=697958&view=diff
==============================================================================
--- incubator/pig/branches/types/CHANGES.txt (original)
+++ incubator/pig/branches/types/CHANGES.txt Mon Sep 22 12:33:26 2008
@@ -221,4 +221,7 @@
     PIG-442: Disambiguated alias after a foreach flatten is not accessible a
     couple of statements after the foreach (sms via olgan)
 
+    PIG-424: nested foreach with flatten and agg gives an error (sms via
+    olgan)
+
 

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java?rev=697958&r1=697957&r2=697958&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java
 (original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java
 Mon Sep 22 12:33:26 2008
@@ -171,7 +171,7 @@
                                                                for(int i = 0; 
i < s.size(); ++i) {
                                     Schema.FieldSchema fs;
                                     try {
-                                        fs = s.getField(i);
+                                        fs = new 
Schema.FieldSchema(s.getField(i));
                                     } catch (ParseException pe) {
                                         throw new 
FrontendException(pe.getMessage());
                                     }

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java?rev=697958&r1=697957&r2=697958&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
 (original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
 Mon Sep 22 12:33:26 2008
@@ -233,7 +233,7 @@
                                 if(null != expOpFs) {
                                     Schema s = expOpFs.schema;
                                     if(null != s) {
-                                        mFieldSchema = 
s.getField(mProjection.get(0));
+                                        mFieldSchema = new 
Schema.FieldSchema(s.getField(mProjection.get(0)));
                                     } else {
                                         mFieldSchema = new 
Schema.FieldSchema(null, DataType.BYTEARRAY);
                                     }
@@ -245,7 +245,7 @@
                                    Schema s = expressionOperator.getSchema();
                                 log.debug("s: " + s);
                                 if(null != s) {
-                                    mFieldSchema = 
s.getField(mProjection.get(0));
+                                    mFieldSchema = new 
Schema.FieldSchema(s.getField(mProjection.get(0)));
                                     log.debug("mFieldSchema alias: " + 
mFieldSchema.alias);
                                     log.debug("mFieldSchema schema: " + 
mFieldSchema.schema);
                                 } else {
@@ -266,7 +266,7 @@
                                     log.debug("Schema s: " + s);
                                     if(null != s) {
                                         if(colNum < s.size()) {
-                                            fss.add(s.getField(colNum));
+                                            fss.add(new 
Schema.FieldSchema(s.getField(colNum)));
                                         } else {
                                             fss.add(new 
Schema.FieldSchema(null, DataType.BYTEARRAY));
                                         }
@@ -279,7 +279,7 @@
                             } else {
                                 Schema s = expressionOperator.getSchema();
                                 if(null != s) {
-                                    fss.add(s.getField(colNum));
+                                    fss.add(new 
Schema.FieldSchema(s.getField(colNum)));
                                 } else {
                                     fss.add(new Schema.FieldSchema(null, 
DataType.BYTEARRAY));
                                 }

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java?rev=697958&r1=697957&r2=697958&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java
 (original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java
 Mon Sep 22 12:33:26 2008
@@ -97,7 +97,7 @@
         if (null != udfSchema) {
             Schema.FieldSchema fs;
             try {
-                fs = udfSchema.getField(0);
+                fs = new Schema.FieldSchema(udfSchema.getField(0));
             } catch (ParseException pe) {
                 throw new FrontendException(pe.getMessage());
             }

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java?rev=697958&r1=697957&r2=697958&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
 (original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
 Mon Sep 22 12:33:26 2008
@@ -137,6 +137,29 @@
             type = t;
         }
 
+        /**
+         * Copy Constructor.
+         * 
+         * @param fs
+         *           Source FieldSchema
+         * 
+         */
+        public FieldSchema(FieldSchema fs)  {
+            if(null != fs) {
+                alias = fs.alias;
+                if(null != fs.schema) {
+                    schema = new Schema(fs.schema);
+                } else {
+                    schema = null;
+                }
+                type = fs.type;
+            } else {
+                alias = null;
+                schema = null;
+                type = DataType.UNKNOWN;
+            }
+        }
+
         /***
          *  Two field schemas are equal if types and schemas
          *  are equal in all levels.
@@ -460,6 +483,41 @@
     }
 
     /**
+     * Copy Constructor.
+     * @param s source schema
+     */
+    public Schema(Schema s) {
+
+        if(null != s) {
+            mFields = new ArrayList<FieldSchema>(s.size());
+            mAliases = new HashMap<String, FieldSchema>();
+            mFieldSchemas = new MultiMap<FieldSchema, String>();
+            try {
+                for (int i = 0; i < s.size(); ++i) {
+                    FieldSchema fs = new FieldSchema(s.getField(i));
+                    if(null != fs) {
+                        mFields.add(fs);
+                        if (fs.alias != null) {
+                            mAliases.put(fs.alias, fs);
+                            if(null != fs) {
+                                mFieldSchemas.put(fs, fs.alias);
+                            }
+                        }
+                    }
+                }
+            } catch (ParseException pe) {
+                mFields = new ArrayList<FieldSchema>();
+                mAliases = new HashMap<String, FieldSchema>();
+                mFieldSchemas = new MultiMap<FieldSchema, String>();
+            }
+        } else {
+            mFields = new ArrayList<FieldSchema>();
+            mAliases = new HashMap<String, FieldSchema>();
+            mFieldSchemas = new MultiMap<FieldSchema, String>();
+        }
+    }
+
+    /**
      * Given an alias name, find the associated FieldSchema.
      * @param alias Alias to look up.
      * @return FieldSchema, or null if no such alias is in this tuple.

Modified: 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=697958&r1=697957&r2=697958&view=diff
==============================================================================
--- 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
 (original)
+++ 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
 Mon Sep 22 12:33:26 2008
@@ -1350,6 +1350,22 @@
         }
     }
 
+    @Test
+    public void testQuery95() throws FrontendException, ParseException {
+        buildPlan("a = load 'myfile' as (name, age, gpa);");
+        buildPlan("b = group a by name;");
+        LogicalPlan lp = buildPlan("c = foreach b {d = order a by $1; generate 
flatten(d), MAX(a.age) as max_age;};");
+        LOForEach foreach = (LOForEach) lp.getLeaves().get(0);
+        LOCogroup cogroup = (LOCogroup) lp.getPredecessors(foreach).get(0);
+        Schema.FieldSchema bagFs = new Schema.FieldSchema("a", 
getSchemaFromString("name: bytearray, age: bytearray, gpa: bytearray"), 
DataType.BAG);
+        Schema.FieldSchema groupFs = new Schema.FieldSchema("group", 
DataType.BYTEARRAY);
+        Schema cogroupExpectedSchema = new Schema();
+        cogroupExpectedSchema.add(groupFs);
+        cogroupExpectedSchema.add(bagFs);
+        assertTrue(Schema.equals(cogroup.getSchema(), cogroupExpectedSchema, 
false, false));
+        assertTrue(Schema.equals(foreach.getSchema(), 
getSchemaFromString("name: bytearray, age: bytearray, gpa: bytearray, max_age: 
double"), false, true));
+    }
+
     private Schema getSchemaFromString(String schemaString) throws 
ParseException {
         return getSchemaFromString(schemaString, DataType.BYTEARRAY);
     }


Reply via email to