Author: daijy
Date: Tue Jan 26 00:34:14 2010
New Revision: 903030
URL: http://svn.apache.org/viewvc?rev=903030&view=rev
Log:
PIG-1189: StoreFunc UDF should ship to the backend automatically without
"register"
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceOper.java
hadoop/pig/trunk/src/org/apache/pig/impl/util/JarManager.java
hadoop/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java
hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld
hadoop/pig/trunk/test/org/apache/pig/test/utils/GenPhyOp.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=903030&r1=903029&r2=903030&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Tue Jan 26 00:34:14 2010
@@ -147,6 +147,9 @@
PIG-1184: PruneColumns optimization does not handle the case of foreach
flatten correctly if flattened bag is not used later (daijy)
+PIG-1189: StoreFunc UDF should ship to the backend automatically without
+"register" (daijy)
+
Release 0.6.0 - Unreleased
INCOMPATIBLE CHANGES
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=903030&r1=903029&r2=903030&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
Tue Jan 26 00:34:14 2010
@@ -732,6 +732,8 @@
storeToMapReduceMap.put(op, curMROp);
nonBlocking(op);
phyToMROpMap.put(op, curMROp);
+ if (op.getSFile()!=null && op.getSFile().getFuncSpec()!=null)
+ curMROp.UDFs.add(op.getSFile().getFuncSpec().toString());
}catch(Exception e){
int errCode = 2034;
String msg = "Error compiling operator " +
op.getClass().getSimpleName();
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceOper.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceOper.java?rev=903030&r1=903029&r2=903030&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceOper.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceOper.java
Tue Jan 26 00:34:14 2010
@@ -20,8 +20,10 @@
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Properties;
+import java.util.Set;
import org.apache.pig.impl.io.FileSpec;
import org.apache.pig.impl.plan.OperatorKey;
@@ -105,7 +107,7 @@
// Sort order for secondary keys;
boolean[] secondarySortOrder;
- public List<String> UDFs;
+ public Set<String> UDFs;
// Indicates if a UDF comparator is used
boolean isUDFComparatorUsed = false;
@@ -142,7 +144,7 @@
mapPlan = new PhysicalPlan();
combinePlan = new PhysicalPlan();
reducePlan = new PhysicalPlan();
- UDFs = new ArrayList<String>();
+ UDFs = new HashSet<String>();
nig = NodeIdGenerator.getGenerator();
scope = k.getScope();
}
Modified: hadoop/pig/trunk/src/org/apache/pig/impl/util/JarManager.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/util/JarManager.java?rev=903030&r1=903029&r2=903030&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/util/JarManager.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/util/JarManager.java Tue Jan 26
00:34:14 2010
@@ -32,6 +32,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.Vector;
import java.util.jar.JarEntry;
import java.util.jar.JarInputStream;
@@ -96,7 +97,7 @@
* @throws ClassNotFoundException
* @throws IOException
*/
- public static void createJar(OutputStream os, List<String> funcs,
PigContext pigContext) throws ClassNotFoundException, IOException {
+ public static void createJar(OutputStream os, Set<String> funcs,
PigContext pigContext) throws ClassNotFoundException, IOException {
Vector<JarListEntry> jarList = new Vector<JarListEntry>();
for(String toSend: pigPackagesToSend) {
addContainingJar(jarList, PigMapReduce.class, toSend, pigContext);
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java?rev=903030&r1=903029&r2=903030&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java Tue Jan 26
00:34:14 2010
@@ -914,8 +914,9 @@
MapReduceOper mrOper = mrPlan.getRoots().get(0);
assertTrue(mrOper.UDFs.size()==2);
- assertTrue(mrOper.UDFs.get(0).equals("BinStorage"));
-
assertTrue(mrOper.UDFs.get(1).equals("org.apache.pig.builtin.PigStorage"));
+ assertTrue(mrOper.UDFs.size()==2);
+ assertTrue(mrOper.UDFs.contains("BinStorage"));
+ assertTrue(mrOper.UDFs.contains("org.apache.pig.builtin.PigStorage"));
}
@Test
Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld?rev=903030&r1=903029&r2=903030&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld
(original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld Tue
Jan 26 00:34:14 2010
@@ -26,7 +26,7 @@
| |
|
|---Load(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage)
- -152
|
- |---MapReduce(20,TestMRCompiler$WeirdComparator,COUNT,SUM) - -145:
+ |---MapReduce(20,SUM,COUNT,TestMRCompiler$WeirdComparator) - -145:
|
Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage)
- -151
| |
| |---New For Each(false,false)[tuple] - -2197807331204639125
@@ -88,4 +88,4 @@
| | |
| | Constant(true) - --6860857982727545176
| |
- | |---Load(DummyFil:DummyLdr) - -737965411848544085
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - -737965411848544085
Modified: hadoop/pig/trunk/test/org/apache/pig/test/utils/GenPhyOp.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/utils/GenPhyOp.java?rev=903030&r1=903029&r2=903030&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/utils/GenPhyOp.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/utils/GenPhyOp.java Tue Jan 26
00:34:14 2010
@@ -762,7 +762,6 @@
public static POStore topStoreOp() {
POStore ret = new POStore(new OperatorKey("", r.nextLong()));
- ret.setSFile(new FileSpec("DummyFil", new FuncSpec("DummyLdr")));
return ret;
}
