Author: knoguchi Date: Tue Jan 11 20:53:23 2022 New Revision: 1896928 URL: http://svn.apache.org/viewvc?rev=1896928&view=rev Log: PIG-5415: [spark] TestScriptLanguage conflict between multiple SparkContext (after spark2.4 upgrade) (knoguchi)
Modified: pig/trunk/CHANGES.txt pig/trunk/build.xml pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java Modified: pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1896928&r1=1896927&r2=1896928&view=diff ============================================================================== --- pig/trunk/CHANGES.txt (original) +++ pig/trunk/CHANGES.txt Tue Jan 11 20:53:23 2022 @@ -104,6 +104,8 @@ OPTIMIZATIONS BUG FIXES +PIG-5415: [spark] TestScriptLanguage conflict between multiple SparkContext (after spark2.4 upgrade) (knoguchi) + PIG-5412: testSkewedJoinOuter spark unit-test failing with ClassNotFoundException (knoguchi) PIG-5404: FLATTEN infers wrong datatype (knoguchi) Modified: pig/trunk/build.xml URL: http://svn.apache.org/viewvc/pig/trunk/build.xml?rev=1896928&r1=1896927&r2=1896928&view=diff ============================================================================== --- pig/trunk/build.xml (original) +++ pig/trunk/build.xml Tue Jan 11 20:53:23 2022 @@ -281,7 +281,7 @@ value="${mvnrepo}/org/codehaus/jackson/jackson-core-asl/${jackson-pig-3039-test.version}/jackson-core-asl-${jackson-pig-3039-test.version}.jar"/> <property name="jackson_mapper_repo_url" value="${mvnrepo}/org/codehaus/jackson/jackson-mapper-asl/${jackson-pig-3039-test.version}/jackson-mapper-asl-${jackson-pig-3039-test.version}.jar"/> - <property name="test.spark.spark_master" value="yarn-client" /> + <property name="test.spark.spark_master" value="yarn" /> <!--this is the naming policy for artifacts we want pulled down--> <property name="ivy.artifact.retrieve.pattern" value="${ant.project.name}/[artifact]-[revision](-[classifier]).[ext]"/> Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java?rev=1896928&r1=1896927&r2=1896928&view=diff ============================================================================== --- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java (original) +++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java Tue Jan 11 20:53:23 2022 @@ -572,7 +572,7 @@ public class SparkLauncher extends Launc } if (sparkContext == null) { String sparkHome = System.getenv("SPARK_HOME"); - if (!master.startsWith("local") && !master.equals("yarn-client")) { + if (!master.startsWith("local") && !master.equals("yarn")) { // Check that we have the Mesos native library and Spark home // are set if (sparkHome == null) {