Author: daijy
Date: Mon Apr 12 21:04:56 2010
New Revision: 933414

URL: http://svn.apache.org/viewvc?rev=933414&view=rev
Log:
PIG-1361: [Zebra] Zebra TableLoader.getSchema() should return the 
projectionSchema specified in the constructor of TableLoader instead of pruned 
proejction by pig

Modified:
    hadoop/pig/branches/branch-0.7/contrib/zebra/CHANGES.txt
    
hadoop/pig/branches/branch-0.7/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/TableLoader.java

Modified: hadoop/pig/branches/branch-0.7/contrib/zebra/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/contrib/zebra/CHANGES.txt?rev=933414&r1=933413&r2=933414&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/contrib/zebra/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.7/contrib/zebra/CHANGES.txt Mon Apr 12 
21:04:56 2010
@@ -18,6 +18,8 @@ Trunk (unreleased changes)
 
   IMPROVEMENTS
 
+    PIG-1361 Zebra TableLoader.getSchema() should return the projectionSchema 
specified in the constructor of TableLoader instead of pruned proejction by pig 
(gauravj via daijy)
+
     PIG-1291 Support of virtual column "source_table" on unsorted table (yanz)
 
     PIG-1315 Implementing OrderedLoadFunc interface for Zebra TableLoader 
(xuefux via yanz)

Modified: 
hadoop/pig/branches/branch-0.7/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/TableLoader.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/TableLoader.java?rev=933414&r1=933413&r2=933414&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.7/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/TableLoader.java
 (original)
+++ 
hadoop/pig/branches/branch-0.7/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/TableLoader.java
 Mon Apr 12 21:04:56 2010
@@ -158,6 +158,7 @@ public class TableLoader extends LoadFun
         schema = TableInputFormat.getSchema( job );
         sorted = true;
         
+        setSortOrder( job );
         setProjection( job );
 
         try {
@@ -169,36 +170,48 @@ public class TableLoader extends LoadFun
         }
     }
 
+    
     /**
-     * This method does more than set projection. For instance, it also try to 
grab sorting info if required.
+     * it processes sortedness of table .
+     * 
+     * @param job
+     * @throws IOException
+     */
+    private void setSortOrder(Job job) throws IOException {
+       Properties properties = UDFContext.getUDFContext().getUDFProperties( 
+              this.getClass(), new String[]{ udfContextSignature } );
+       boolean requireGlobalOrder = "true".equals(properties.getProperty( 
UDFCONTEXT_GLOBAL_SORTING));
+       if (requireGlobalOrder && !sorted)
+         throw new IOException("Global sorting can be only asked on table 
loaded as sorted");
+       if( sorted ) {
+           SplitMode splitMode = 
+             requireGlobalOrder ? SplitMode.GLOBALLY_SORTED : 
SplitMode.LOCALLY_SORTED;
+           TableInputFormat.setSplitMode(job, splitMode, null);
+           sortInfo = TableInputFormat.getSortInfo( job );
+       }       
+    }
+    
+    
+    /**
+     * This method sets projection.
      * 
      * @param job
      * @throws IOException
      */
     private void setProjection(Job job) throws IOException {
       Properties properties = UDFContext.getUDFContext().getUDFProperties( 
-          this.getClass(), new String[]{ udfContextSignature } );
-      boolean requireGlobalOrder = "true".equals(properties.getProperty( 
UDFCONTEXT_GLOBAL_SORTING));
-      if (requireGlobalOrder && !sorted)
-        throw new IOException("Global sorting can be only asked on table 
loaded as sorted");
-        if( sorted ) {
-            SplitMode splitMode = 
-              requireGlobalOrder ? SplitMode.GLOBALLY_SORTED : 
SplitMode.LOCALLY_SORTED;
-            TableInputFormat.setSplitMode(job, splitMode, null);
-            sortInfo = TableInputFormat.getSortInfo( job );
-        }
-        
-        try {
-            String prunedProjStr = properties.getProperty( 
UDFCONTEXT_PROJ_STRING );
-            
-            if( prunedProjStr != null ) {
-                TableInputFormat.setProjection( job, prunedProjStr );
-            } else if( projectionString != null ) {              
-                TableInputFormat.setProjection( job, projectionString );
-            }
-        } catch (ParseException ex) {
-            throw new IOException( "Schema parsing failed : " + 
ex.getMessage() );
-        }
+          this.getClass(), new String[]{ udfContextSignature } );   
+      try {
+          String prunedProjStr = properties.getProperty( 
UDFCONTEXT_PROJ_STRING );
+          
+          if( prunedProjStr != null ) {
+              TableInputFormat.setProjection( job, prunedProjStr );
+          } else if( projectionString != null ) {              
+              TableInputFormat.setProjection( job, projectionString );
+          }
+      } catch (ParseException ex) {
+          throw new IOException( "Schema parsing failed : " + ex.getMessage() 
);
+      }
     }
 
     private KeyGenerator makeKeyBuilder(byte[] elems) {
@@ -283,6 +296,7 @@ public class TableLoader extends LoadFun
 
          // The following obviously goes beyond of set location, but this is 
the only place that we
          // can do and it's suggested by Pig team.
+         setSortOrder( job );
          setProjection( job );
      }
 
@@ -318,10 +332,25 @@ public class TableLoader extends LoadFun
              }
          }
          
-         setProjection( job );
+         // This is needed as it does a check if a unsorted table is loaded as 
sorted
+         // It fails if unosrted table is loaded as sorted
+         setSortOrder( job );
+         
+         /*
+         As per pig team any changes to this job object will be thrown away.
+         getSchema is needed to return the projectionSchema for the projection
+         string specified in TableLoader constructor. So, projectionString is 
used
+         here. However, setLocation() calls setPojection() because that is 
called after
+         projectionPruning and needs to read projection string from UDFCONTEXT
+         That also sets/calls TableInputFormat.setProjection(job, 
$prunedProj). But the 
+         job object here in getSchema() is a different copy from setLocation() 
and hence 
+         the changes will not be overridden as per PIG TEAM.  
+        */
 
          projectionSchema = tableSchema;
          try {
+                if(projectionString != null)
+                        TableInputFormat.setProjection(job, projectionString); 
                         
              Projection projection = new 
org.apache.hadoop.zebra.types.Projection( tableSchema, 
                      TableInputFormat.getProjection( job ) );
              projectionSchema = projection.getProjectionSchema();


Reply via email to