Author: daijy
Date: Fri Jul 10 23:31:55 2009
New Revision: 793147
URL: http://svn.apache.org/viewvc?rev=793147&view=rev
Log:
PIG-881: Pig should ship load udfs to the backend
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
hadoop/pig/trunk/src/org/apache/pig/impl/util/JarManager.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=793147&r1=793146&r2=793147&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Fri Jul 10 23:31:55 2009
@@ -40,6 +40,8 @@
BUG FIXES
+ PIG-881: Pig should ship load udfs to the backend (daijy)
+
PIG-876: limit changes order of order-by to ascending (daijy)
PIG-851: Map type used as return type in UDFs not recognized at all times
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=793147&r1=793146&r2=793147&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
Fri Jul 10 23:31:55 2009
@@ -333,6 +333,11 @@
//operator op. Also this should be added to the MRPlan.
curMROp = getMROp();
curMROp.mapPlan.add(op);
+ if (op !=null && op instanceof POLoad)
+ {
+ if (((POLoad)op).getLFile()!=null &&
((POLoad)op).getLFile().getFuncSpec()!=null)
+
curMROp.UDFs.add(((POLoad)op).getLFile().getFuncSpec().toString());
+ }
MRPlan.add(curMROp);
return;
}
Modified: hadoop/pig/trunk/src/org/apache/pig/impl/util/JarManager.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/util/JarManager.java?rev=793147&r1=793146&r2=793147&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/util/JarManager.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/util/JarManager.java Fri Jul 10
23:31:55 2009
@@ -39,13 +39,15 @@
import java.util.zip.ZipEntry;
//import
org.apache.pig.backend.hadoop.executionengine.mapreduceExec.PigMapReduce;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.pig.impl.PigContext;
import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
public class JarManager {
-
+ private static Log log = LogFactory.getLog(JarManager.class);
/**
* A container class to track the Jar files that need to be merged
together to submit to Hadoop.
*/
@@ -104,9 +106,9 @@
for (String func: funcs) {
Class clazz = pigContext.getClassForAlias(func);
if (clazz != null) {
- if (pigClassLoader == clazz.getClassLoader()) {
+ /*if (pigClassLoader == clazz.getClassLoader()) {
continue;
- }
+ }*/
addContainingJar(jarList, clazz, null, pigContext);
}
}
@@ -236,7 +238,11 @@
if (pigContext.skipJars.contains(jar) && prefix == null)
return;
if (jar == null)
- throw new RuntimeException("Couldn't find the jar for " +
clazz.getName());
+ {
+ //throw new RuntimeException("Couldn't find the jar for " +
clazz.getName());
+ log.warn("Couldn't find the jar for " + clazz.getName() + ", skip
it");
+ return;
+ }
JarListEntry jarListEntry = new JarListEntry(jar, prefix);
if (!jarList.contains(jarListEntry))
jarList.add(jarListEntry);
