Author: olga Date: Mon Dec 21 19:52:08 2009 New Revision: 892943 URL: http://svn.apache.org/viewvc?rev=892943&view=rev Log: PIG-1149: Allow instantiation of SampleLoaders with parametrized LoadFuncs (dvryaboy via olgan)
Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/impl/builtin/SampleLoader.java hadoop/pig/trunk/test/org/apache/pig/test/TestPoissonSampleLoader.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=892943&r1=892942&r2=892943&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Mon Dec 21 19:52:08 2009 @@ -24,6 +24,9 @@ IMPROVEMENTS +PIG-1149: Allow instantiation of SampleLoaders with parametrized LoadFuncs +(dvryaboy via olgan) + PIG-1162: Pig 0.6.0 - UDF doc (chandec via olgan) PIG-1163: Pig/Zebra 0.6.0 release (chandec via olgan) Modified: hadoop/pig/trunk/src/org/apache/pig/impl/builtin/SampleLoader.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/builtin/SampleLoader.java?rev=892943&r1=892942&r2=892943&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/impl/builtin/SampleLoader.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/impl/builtin/SampleLoader.java Mon Dec 21 19:52:08 2009 @@ -35,6 +35,7 @@ import org.apache.pig.impl.logicalLayer.FrontendException; import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.impl.util.Pair; +import org.mortbay.log.Log; /** * Abstract class that specifies the interface for sample loaders @@ -49,7 +50,8 @@ public SampleLoader(String funcSpec) { - loader = (SamplableLoader)PigContext.instantiateFuncFromSpec(funcSpec); + funcSpec = funcSpec.replaceAll("\\\\'", "'"); + loader = (SamplableLoader)PigContext.instantiateFuncFromSpec(funcSpec); } public void setNumSamples(int n) { Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPoissonSampleLoader.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPoissonSampleLoader.java?rev=892943&r1=892942&r2=892943&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestPoissonSampleLoader.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestPoissonSampleLoader.java Mon Dec 21 19:52:08 2009 @@ -24,6 +24,8 @@ import junit.framework.Assert; import junit.framework.TestCase; +import org.junit.Test; +import org.mortbay.log.Log; import org.apache.pig.EvalFunc; import org.apache.pig.ExecType; @@ -56,11 +58,10 @@ private MiniCluster cluster = MiniCluster.buildCluster(); public TestPoissonSampleLoader() throws ExecException, IOException{ - pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties()); + pigServer = new PigServer(ExecType.LOCAL, cluster.getProperties()); pigServer.getPigContext().getProperties().setProperty("pig.skewedjoin.reduce.maxtuple", "5"); pigServer.getPigContext().getProperties().setProperty("pig.skewedjoin.reduce.memusage", "0.0001"); pigServer.getPigContext().getProperties().setProperty("mapred.child.java.opts", "-Xmx512m"); - pigServer.getPigContext().getProperties().setProperty("pig.mapsplits.count", "5"); } @@ -75,11 +76,11 @@ int k = 0; for(int j=0; j<100; j++) { - w.println("100\tapple1\taaa" + k); + w.println("100:apple1:aaa" + k); k++; - w.println("200\torange1\tbbb" + k); + w.println("200:orange1:bbb" + k); k++; - w.println("300\tstrawberry\tccc" + k); + w.println("300:strawberry:ccc" + k); k++; } @@ -115,5 +116,14 @@ fail("Compute samples returned the wrong number of samples: " + ps.getNumSamples() + " instead of 3"); } } - -} \ No newline at end of file + + @Test + public void testInstantiation() throws IOException { + Log.info("A = Load '"+INPUT_FILE1+"' Using PoissonSampleLoader('PigStorage(\\\\\\':\\\\\\')', '100')"); + pigServer.registerQuery("A = Load '"+INPUT_FILE1+"' Using PoissonSampleLoader('PigStorage(\\\\\\':\\\\\\')', '100');"); + Iterator<Tuple> iter = pigServer.openIterator("A"); + assertTrue(iter.hasNext()); + assertEquals(iter.next().size(), 4); + } + +}