Author: pradeepkth
Date: Tue Aug  4 17:12:01 2009
New Revision: 800877

URL: http://svn.apache.org/viewvc?rev=800877&view=rev
Log:
PIG-901: InputSplit (SliceWrapper) created by Pig is big in size due to 
serialized PigContext (pradeepkth)

Modified:
    hadoop/pig/branches/branch-0.3/CHANGES.txt
    hadoop/pig/branches/branch-0.3/src/org/apache/pig/ExecType.java
    
hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
    
hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java

Modified: hadoop/pig/branches/branch-0.3/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/CHANGES.txt?rev=800877&r1=800876&r2=800877&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.3/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.3/CHANGES.txt Tue Aug  4 17:12:01 2009
@@ -22,6 +22,9 @@
 
 BUG FIXES 
 
+PIG-901: InputSplit (SliceWrapper) created by Pig is big in size due to
+serialized PigContext (pradeepkth)
+
 PIG-878: Pig is returning too many blocks in the input split. (arunc via 
gates).
 
 Release 0.3.0 - 06/16/09

Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/ExecType.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/ExecType.java?rev=800877&r1=800876&r2=800877&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.3/src/org/apache/pig/ExecType.java (original)
+++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/ExecType.java Tue Aug  4 
17:12:01 2009
@@ -18,10 +18,12 @@
 
 package org.apache.pig;
 
+import java.io.Serializable;
+
 /**
  * The type of query execution
  */
-public enum ExecType {
+public enum ExecType implements Serializable {
     /**
      * Run everything on the local machine
      */

Modified: 
hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java?rev=800877&r1=800876&r2=800877&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
 (original)
+++ 
hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
 Tue Aug  4 17:12:01 2009
@@ -233,7 +233,7 @@
                                }
                                Slice[] pigs = spec.getSlicer().slice(store, 
spec.getFileName());
                                for (Slice split : pigs) {
-                                   splits.add(new SliceWrapper(split, 
pigContext, i, fs, inpTargets.get(i)));
+                                   splits.add(new SliceWrapper(split, 
pigContext.getExecType(), i, fs, inpTargets.get(i)));
                                }
             } catch (ExecException ee) {
                throw ee;

Modified: 
hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java?rev=800877&r1=800876&r2=800877&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java
 (original)
+++ 
hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java
 Tue Aug  4 17:12:01 2009
@@ -60,7 +60,7 @@
 public class SliceWrapper implements InputSplit {
 
     private int index;
-    private PigContext pigContext;
+    private ExecType execType;
     private Slice wrapped;
     private transient FileSystem fs;// transient so it isn't serialized
     private transient JobConf lastConf;
@@ -70,9 +70,9 @@
         // for deserialization
     }
 
-    public SliceWrapper(Slice slice, PigContext context, int index, FileSystem 
fs, ArrayList<OperatorKey> targetOps) {
+    public SliceWrapper(Slice slice, ExecType execType, int index, FileSystem 
fs, ArrayList<OperatorKey> targetOps) {
         this.wrapped = slice;
-        this.pigContext = context;
+        this.execType = execType;
         this.index = index;
         this.fs = fs;
         this.targetOps = targetOps;
@@ -122,7 +122,7 @@
         DataStorage store = new 
HDataStorage(ConfigurationUtil.toProperties(job));
         // if the execution is against Mapred DFS, set
         // working dir to /user/<userid>
-        if(pigContext.getExecType() == ExecType.MAPREDUCE)
+        if(execType == ExecType.MAPREDUCE)
             store.setActiveContainer(store.asContainer("/user/" + 
job.getUser()));
         wrapped.init(store);
         
@@ -165,7 +165,7 @@
     }
 
     public void readFields(DataInput is) throws IOException {
-        pigContext = (PigContext) readObject(is);
+        execType = (ExecType) readObject(is);
         targetOps = (ArrayList<OperatorKey>) readObject(is);
         index = is.readInt();
         wrapped = (Slice) readObject(is);
@@ -192,7 +192,7 @@
     }
 
     public void write(DataOutput os) throws IOException {
-        writeObject(pigContext, os);
+        writeObject(execType, os);
         writeObject(targetOps, os);
         os.writeInt(index);
         writeObject(wrapped, os);


Reply via email to