svn commit: r892943 - in /hadoop/pig/trunk: CHANGES.txt src/org/apache/pig/impl/builtin/SampleLoader.java test/org/apache/pig/test/TestPoissonSampleLoader.java

[olga]

Author: olga
Date: Mon Dec 21 19:52:08 2009
New Revision: 892943

URL: http://svn.apache.org/viewvc?rev=892943&view=rev
Log:
PIG-1149: Allow instantiation of SampleLoaders with parametrized LoadFuncs
(dvryaboy via olgan)

Modified:
    hadoop/pig/trunk/CHANGES.txt
    hadoop/pig/trunk/src/org/apache/pig/impl/builtin/SampleLoader.java
    hadoop/pig/trunk/test/org/apache/pig/test/TestPoissonSampleLoader.java

Modified: hadoop/pig/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=892943&r1=892942&r2=892943&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Mon Dec 21 19:52:08 2009
@@ -24,6 +24,9 @@
 
 IMPROVEMENTS
 
+PIG-1149: Allow instantiation of SampleLoaders with parametrized LoadFuncs
+(dvryaboy via olgan)
+
 PIG-1162: Pig 0.6.0 - UDF doc (chandec via olgan)
 
 PIG-1163: Pig/Zebra 0.6.0 release (chandec via olgan)

Modified: hadoop/pig/trunk/src/org/apache/pig/impl/builtin/SampleLoader.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/builtin/SampleLoader.java?rev=892943&r1=892942&r2=892943&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/builtin/SampleLoader.java 
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/builtin/SampleLoader.java Mon Dec 
21 19:52:08 2009
@@ -35,6 +35,7 @@
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.util.Pair;
+import org.mortbay.log.Log;
 
 /**
  * Abstract class that specifies the interface for sample loaders
@@ -49,7 +50,8 @@
 
     
     public SampleLoader(String funcSpec) {
-       loader = (SamplableLoader)PigContext.instantiateFuncFromSpec(funcSpec);
+        funcSpec = funcSpec.replaceAll("\\\\'", "'");
+        loader = (SamplableLoader)PigContext.instantiateFuncFromSpec(funcSpec);
     }
     
     public void setNumSamples(int n) {

Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPoissonSampleLoader.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPoissonSampleLoader.java?rev=892943&r1=892942&r2=892943&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPoissonSampleLoader.java 
(original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPoissonSampleLoader.java Mon 
Dec 21 19:52:08 2009
@@ -24,6 +24,8 @@
 
 import junit.framework.Assert;
 import junit.framework.TestCase;
+import org.junit.Test;
+import org.mortbay.log.Log;
 
 import org.apache.pig.EvalFunc;
 import org.apache.pig.ExecType;
@@ -56,11 +58,10 @@
     private MiniCluster cluster = MiniCluster.buildCluster();
     
     public TestPoissonSampleLoader() throws ExecException, IOException{
-        pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+        pigServer = new PigServer(ExecType.LOCAL, cluster.getProperties());
         
pigServer.getPigContext().getProperties().setProperty("pig.skewedjoin.reduce.maxtuple",
 "5");     
         
pigServer.getPigContext().getProperties().setProperty("pig.skewedjoin.reduce.memusage",
 "0.0001");
         
pigServer.getPigContext().getProperties().setProperty("mapred.child.java.opts", 
"-Xmx512m");
-
         
pigServer.getPigContext().getProperties().setProperty("pig.mapsplits.count", 
"5");
     }
     
@@ -75,11 +76,11 @@
                
        int k = 0;
        for(int j=0; j<100; j++) {                              
-               w.println("100\tapple1\taaa" + k);
+               w.println("100:apple1:aaa" + k);
            k++;
-           w.println("200\torange1\tbbb" + k);
+           w.println("200:orange1:bbb" + k);
            k++;
-           w.println("300\tstrawberry\tccc" + k);
+           w.println("300:strawberry:ccc" + k);
            k++;                            
        }
        
@@ -115,5 +116,14 @@
                fail("Compute samples returned the wrong number of samples: " + 
ps.getNumSamples() + " instead of 3");
         }
     }
-       
-}
\ No newline at end of file
+
+    @Test
+    public void testInstantiation() throws IOException {
+        Log.info("A = Load '"+INPUT_FILE1+"' Using 
PoissonSampleLoader('PigStorage(\\\\\\':\\\\\\')', '100')");
+        pigServer.registerQuery("A = Load '"+INPUT_FILE1+"' Using 
PoissonSampleLoader('PigStorage(\\\\\\':\\\\\\')', '100');");
+        Iterator<Tuple> iter = pigServer.openIterator("A");
+        assertTrue(iter.hasNext());
+        assertEquals(iter.next().size(), 4);
+    }
+    
+}