Author: rohini
Date: Tue Mar 22 10:45:34 2016
New Revision: 1736181
URL: http://svn.apache.org/viewvc?rev=1736181&view=rev
Log:
PIG-4845: Parallel instantiation of classes in Tez cause tasks to fail (rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/impl/PigContext.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1736181&r1=1736180&r2=1736181&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Mar 22 10:45:34 2016
@@ -103,6 +103,8 @@ PIG-4639: Add better parser for Apache H
BUG FIXES
+PIG-4845: Parallel instantiation of classes in Tez cause tasks to fail (rohini)
+
PIG-4841: Inline-op with schema declaration fails with syntax error (knoguchi)
PIG-4832: Fix TestPrumeColumn NPE failure (kellyzly via daijy)
Modified: pig/trunk/src/org/apache/pig/impl/PigContext.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/PigContext.java?rev=1736181&r1=1736180&r2=1736181&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/PigContext.java (original)
+++ pig/trunk/src/org/apache/pig/impl/PigContext.java Tue Mar 22 10:45:34 2016
@@ -71,13 +71,13 @@ public class PigContext implements Seria
private static final long serialVersionUID = 1L;
private static final Log log = LogFactory.getLog(PigContext.class);
+ private static Object instantiationLock = new Object();
public static final String JOB_NAME = "jobName";
public static final String JOB_NAME_PREFIX= "PigLatin";
public static final String JOB_PRIORITY = "jobPriority";
public static final String PIG_CMD_ARGS_REMAINDERS =
"pig.cmd.args.remainders";
-
/* NOTE: we only serialize some of the stuff
*
*(to make it smaller given that it's not all needed on the Hadoop side,
@@ -723,26 +723,41 @@ public class PigContext implements Seria
throw new RuntimeException("Cannot instantiate: " + className,
ioe) ;
}
- try {
- // Do normal instantiation
- if (args != null && args.length > 0) {
- Class paramTypes[] = new Class[args.length];
- for (int i = 0; i < paramTypes.length; i++) {
- paramTypes[i] = String.class;
+ // OptionBuilder is not thread-safe and HBaseStorage, elephantbird
SequenceFileConfig, etc
+ // use them in constructor. This leads to NoSuchMethodException,
UnrecognizedOptionException etc
+ // when processor, inputs and outputs are initialized in parallel in
Tez
+ synchronized (instantiationLock) {
+ try {
+ // Do normal instantiation
+ if (args != null && args.length > 0) {
+ Class paramTypes[] = new Class[args.length];
+ for (int i = 0; i < paramTypes.length; i++) {
+ paramTypes[i] = String.class;
+ }
+ Constructor c = objClass.getConstructor(paramTypes);
+ ret = c.newInstance((Object[])args);
+ } else {
+ ret = objClass.newInstance();
}
- Constructor c = objClass.getConstructor(paramTypes);
- ret = c.newInstance((Object[])args);
- } else {
- ret = objClass.newInstance();
}
- }
- catch(NoSuchMethodException nme) {
- // Second chance. Try with var arg constructor
- try {
- Constructor c = objClass.getConstructor(String[].class);
- Object[] wrappedArgs = new Object[1] ;
- wrappedArgs[0] = args ;
- ret = c.newInstance(wrappedArgs);
+ catch(NoSuchMethodException nme) {
+ // Second chance. Try with var arg constructor
+ try {
+ Constructor c = objClass.getConstructor(String[].class);
+ Object[] wrappedArgs = new Object[1] ;
+ wrappedArgs[0] = args ;
+ ret = c.newInstance(wrappedArgs);
+ }
+ catch(Throwable e){
+ // bad luck
+ StringBuilder sb = new StringBuilder();
+ sb.append("could not instantiate '");
+ sb.append(className);
+ sb.append("' with arguments '");
+ sb.append(Arrays.toString(args));
+ sb.append("'");
+ throw new RuntimeException(sb.toString(), e);
+ }
}
catch(Throwable e){
// bad luck
@@ -754,18 +769,8 @@ public class PigContext implements Seria
sb.append("'");
throw new RuntimeException(sb.toString(), e);
}
+ return ret;
}
- catch(Throwable e){
- // bad luck
- StringBuilder sb = new StringBuilder();
- sb.append("could not instantiate '");
- sb.append(className);
- sb.append("' with arguments '");
- sb.append(Arrays.toString(args));
- sb.append("'");
- throw new RuntimeException(sb.toString(), e);
- }
- return ret;
}
public static Object instantiateFuncFromSpec(String funcSpec) {
