svn commit: r595880 - in /incubator/pig/trunk: CHANGES.txt src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java

[olga]

Author: olga
Date: Fri Nov 16 16:21:46 2007
New Revision: 595880

URL: http://svn.apache.org/viewvc?rev=595880&view=rev
Log:
added binary comparator; PIG-8

Modified:
    incubator/pig/trunk/CHANGES.txt
    
incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java

Modified: incubator/pig/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=595880&r1=595879&r2=595880&view=diff
==============================================================================
--- incubator/pig/trunk/CHANGES.txt (original)
+++ incubator/pig/trunk/CHANGES.txt Fri Nov 16 16:21:46 2007
@@ -18,4 +18,4 @@
 
        PIG-23 Made pig work with java 1.5. (milindb via gates)
 
-
+       PIG-8 added binary comparator (olgan)

Modified: 
incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java?rev=595880&r1=595879&r2=595880&view=diff
==============================================================================
--- 
incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java
 (original)
+++ 
incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java
 Fri Nov 16 16:21:46 2007
@@ -40,6 +40,7 @@
 
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.UTF8;
+import org.apache.hadoop.io.WritableComparator;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.TaskReport;
 import org.apache.hadoop.mapred.JobClient;
@@ -63,6 +64,16 @@
         numMRJobs = numMRJobsIn;
         mrJobNumber = 0;
     }
+
+    public static class PigWritableComparator extends WritableComparator {
+        public PigWritableComparator() {
+            super(Tuple.class);
+        }
+
+        public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int 
l2){
+            return WritableComparator.compareBytes(b1, s1, l1, b2, s2, l2);
+           }
+    }
         
     static Random rand = new Random();
 
@@ -136,7 +147,11 @@
                        conf.setCombinerClass(PigCombine.class);
                if (pom.quantilesFile!=null){
                        conf.set("pig.quantilesFile", pom.quantilesFile);
-               }
+                }
+                else{
+                    // this is not a sort job - can use byte comparison to 
speed up processing
+                    
conf.setOutputKeyComparatorClass(PigWritableComparator.class);                  
                   
+                }
                if (pom.partitionFunction!=null){
                        conf.setPartitionerClass(SortPartitioner.class);
                }