Author: zly Date: Wed Apr 12 01:52:15 2017 New Revision: 1791059 URL: http://svn.apache.org/viewvc?rev=1791059&view=rev Log: PIG-5176:Several ComputeSpec test cases fail(Nandor via Liyun)
Modified: pig/branches/spark/src/org/apache/pig/PigConfiguration.java pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java Modified: pig/branches/spark/src/org/apache/pig/PigConfiguration.java URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/PigConfiguration.java?rev=1791059&r1=1791058&r2=1791059&view=diff ============================================================================== --- pig/branches/spark/src/org/apache/pig/PigConfiguration.java (original) +++ pig/branches/spark/src/org/apache/pig/PigConfiguration.java Wed Apr 12 01:52:15 2017 @@ -482,6 +482,11 @@ public class PigConfiguration { public static final String PIG_LOG_TRACE_ID = "pig.log.trace.id"; /** + * Use Netty file server for Pig on Spark, true or false, default value is false + */ + public static final String PIG_SPARK_USE_NETTY_FILESERVER = "pig.spark.rpc.useNettyFileServer"; + + /** * @deprecated use {@link #PIG_LOG_TRACE_ID} instead. Will be removed in Pig 0.18 */ public static final String CALLER_ID = PIG_LOG_TRACE_ID; Modified: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java?rev=1791059&r1=1791058&r2=1791059&view=diff ============================================================================== --- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java (original) +++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java Wed Apr 12 01:52:15 2017 @@ -559,6 +559,12 @@ public class SparkLauncher extends Launc sparkConf.setMaster(master); sparkConf.setAppName("PigOnSpark:" + pigCtxtProperties.getProperty(PigContext.JOB_NAME)); + // On Spark 1.6, Netty file server doesn't allow adding the same file with the same name twice + // This is a problem for streaming using a script + explicit ship the same script combination (PIG-5134) + // HTTP file server doesn't have this restriction, it overwrites the file if added twice + String useNettyFileServer = pigCtxtProperties.getProperty(PigConfiguration.PIG_SPARK_USE_NETTY_FILESERVER, "false"); + sparkConf.set("spark.rpc.useNettyFileServer", useNettyFileServer); + if (sparkHome != null && !sparkHome.isEmpty()) { sparkConf.setSparkHome(sparkHome); } else {