Author: daijy
Date: Mon May 17 17:19:45 2010
New Revision: 945246

URL: http://svn.apache.org/viewvc?rev=945246&view=rev
Log:
PIG-1415: LoadFunc signature is not correct in LoadFunc.getSchema sometimes

Added:
    
hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/PigStorageWithSchema.java
Modified:
    hadoop/pig/branches/branch-0.7/CHANGES.txt
    
hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/LOLoad.java
    
hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestLogicalPlanBuilder.java

Modified: hadoop/pig/branches/branch-0.7/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/CHANGES.txt?rev=945246&r1=945245&r2=945246&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.7/CHANGES.txt Mon May 17 17:19:45 2010
@@ -189,6 +189,8 @@ OPTIMIZATIONS
 
 BUG FIXES
 
+PIG-1415: LoadFunc signature is not correct in LoadFunc.getSchema sometimes 
(daijy)
+
 PIG-1403: Make Pig work with remote HDFS in secure mode (daijy)
 
 PIG-1391: pig unit tests leave behind files in temp directory because 

Modified: 
hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/LOLoad.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/LOLoad.java?rev=945246&r1=945245&r2=945246&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/LOLoad.java 
(original)
+++ 
hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/LOLoad.java 
Mon May 17 17:19:45 2010
@@ -59,6 +59,7 @@ public class LOLoad extends RelationalOp
     transient private Configuration conf;
     private static Log log = LogFactory.getLog(LOLoad.class);
     private Schema mDeterminedSchema = null;
+    private Schema scriptSchema = null;
     private RequiredFieldList requiredFieldList;
 
     private boolean mDeterminedSchemaCached = false;
@@ -149,6 +150,20 @@ public class LOLoad extends RelationalOp
                 if(null == mDeterminedSchema) {
                     mSchema = determineSchema();
                 }
+                if (mSchema == null) {
+                    log.debug("Operator schema is null; Setting it to new 
schema");
+                    mSchema = scriptSchema;
+                } else {
+                    log.debug("Reconciling schema");
+                    log.debug("mSchema: " + mSchema + " schema: " + 
scriptSchema);
+                    try {
+                        mSchema = mSchema.mergePrefixSchema(scriptSchema, 
true, true);
+                    } catch (SchemaMergeException e) {
+                        int errCode = 1019;
+                        String msg = "Unable to merge schemas";
+                        throw new FrontendException(msg, errCode, 
PigException.INPUT, false, null, e);
+                    }
+                }
                 mIsSchemaComputed = true;
             } catch (IOException ioe) {
                 int errCode = 1018;
@@ -182,27 +197,7 @@ public class LOLoad extends RelationalOp
      */
     @Override
     public void setSchema(Schema schema) throws FrontendException {
-        // In general, operators don't generate their schema until they're
-        // asked, so ask them to do it.
-        try {
-            getSchema();
-        } catch (FrontendException ioe) {
-            // It's fine, it just means we don't have a schema yet.
-        }
-        if (mSchema == null) {
-            log.debug("Operator schema is null; Setting it to new schema");
-            mSchema = schema;
-        } else {
-            log.debug("Reconciling schema");
-            log.debug("mSchema: " + mSchema + " schema: " + schema);
-            try {
-                mSchema = mSchema.mergePrefixSchema(schema, true, true);
-            } catch (SchemaMergeException e) {
-                int errCode = 1019;
-                String msg = "Unable to merge schemas";
-                throw new FrontendException(msg, errCode, PigException.INPUT, 
false, null, e);
-            }
-        }
+        scriptSchema = schema;
     }
     
 

Added: 
hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/PigStorageWithSchema.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/PigStorageWithSchema.java?rev=945246&view=auto
==============================================================================
--- 
hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/PigStorageWithSchema.java
 (added)
+++ 
hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/PigStorageWithSchema.java
 Mon May 17 17:19:45 2010
@@ -0,0 +1,46 @@
+package org.apache.pig.test;
+
+import java.io.IOException;
+
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.pig.Expression;
+import org.apache.pig.LoadMetadata;
+import org.apache.pig.ResourceSchema;
+import org.apache.pig.ResourceStatistics;
+import org.apache.pig.builtin.PigStorage;
+
+public class PigStorageWithSchema extends PigStorage implements LoadMetadata {
+    private String signature;
+
+    @Override
+    public String[] getPartitionKeys(String location, Job job)
+            throws IOException {
+        return null;
+    }
+
+    @Override
+    public ResourceSchema getSchema(String location, Job job)
+            throws IOException {
+        return null;
+    }
+
+    @Override
+    public ResourceStatistics getStatistics(String location, Job job)
+            throws IOException {
+        return null;
+    }
+
+    @Override
+    public void setPartitionFilter(Expression partitionFilter)
+            throws IOException {
+    }
+    
+    @Override
+    public void setUDFContextSignature(String signature) {
+        this.signature = signature;
+    }
+    
+    public String getUDFContextSignature() {
+        return signature;
+    }
+}

Modified: 
hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=945246&r1=945245&r2=945246&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestLogicalPlanBuilder.java
 (original)
+++ 
hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestLogicalPlanBuilder.java
 Mon May 17 17:19:45 2010
@@ -2144,6 +2144,14 @@ public class TestLogicalPlanBuilder exte
                 "not occur", true, exceptionThrown);
     }
     
+    @Test
+    public void testLoaderSignature() {
+        LogicalPlan plan = buildPlan(" a = load '1.txt' using 
org.apache.pig.test.PigStorageWithSchema() as (a0:int, a1:int);");
+        
assertTrue(((PigStorageWithSchema)((LOLoad)plan.getLeaves().get(0)).getLoadFunc()).getUDFContextSignature().equals("a"));
+        plan = buildPlan(" b = load '1.txt' using 
org.apache.pig.test.PigStorageWithSchema();");
+        
assertTrue(((PigStorageWithSchema)((LOLoad)plan.getLeaves().get(0)).getLoadFunc()).getUDFContextSignature().equals("b"));
+    }
+    
     private void printPlan(LogicalPlan lp) {
         LOPrinter graphPrinter = new LOPrinter(System.err, lp);
         System.err.println("Printing the logical plan");


Reply via email to