Author: xuefu
Date: Fri Dec 18 05:31:19 2015
New Revision: 1720722
URL: http://svn.apache.org/viewvc?rev=1720722&view=rev
Log:
PIG-4754: Fix UT failures in TestScriptLanguage (Xianda via Xuefu)
Modified:
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
Modified:
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
URL:
http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java?rev=1720722&r1=1720721&r2=1720722&view=diff
==============================================================================
---
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
(original)
+++
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
Fri Dec 18 05:31:19 2015
@@ -371,17 +371,20 @@ public class SparkLauncher extends Launc
&& jarFile.exists()) {
return;
}
- if (localFile.exists()) {
- LOG.info(String.format(
- "jar file %s exists, ready to delete",
- localFile.getAbsolutePath()));
- localFile.delete();
- } else {
- LOG.info(String.format("jar file %s not exists,",
- localFile.getAbsolutePath()));
+ // When multiple threads start SparkLauncher, delete/copy actions
should be in a critical section
+ synchronized(SparkLauncher.class) {
+ if (localFile.exists()) {
+ LOG.info(String.format(
+ "jar file %s exists, ready to delete",
+ localFile.getAbsolutePath()));
+ localFile.delete();
+ } else {
+ LOG.info(String.format("jar file %s not exists,",
+ localFile.getAbsolutePath()));
+ }
+ Files.copy(Paths.get(new
Path(jarFile.getAbsolutePath()).toString()),
+ Paths.get(localFile.getAbsolutePath()));
}
- Files.copy(Paths.get(new
Path(jarFile.getAbsolutePath()).toString()),
- Paths.get(localFile.getAbsolutePath()));
} else {
sparkContext.addFile(jarFile.toURI().toURL()
.toExternalForm());
@@ -431,7 +434,11 @@ public class SparkLauncher extends Launc
return sparkPlan;
}
- private static void startSparkIfNeeded(PigContext pc) throws PigException {
+ /**
+ * Only one SparkContext may be active per JVM (SPARK-2243). When multiple
threads start SparkLaucher,
+ * the static member sparkContext should be initialized only once
+ */
+ private static synchronized void startSparkIfNeeded(PigContext pc) throws
PigException {
if (sparkContext == null) {
String master = null;
if (pc.getExecType().isLocal()) {
