Author: xuefu
Date: Wed Apr 22 01:22:38 2015
New Revision: 1675242
URL: http://svn.apache.org/r1675242
Log:
PIG-4489: Enable local mode tests for Spark engine (Mohit via Xuefu)
Added:
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLocalExecType.java
pig/branches/spark/test/spark-local-tests
Modified:
pig/branches/spark/build.xml
pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
pig/branches/spark/test/org/apache/pig/test/Util.java
Modified: pig/branches/spark/build.xml
URL:
http://svn.apache.org/viewvc/pig/branches/spark/build.xml?rev=1675242&r1=1675241&r2=1675242&view=diff
==============================================================================
--- pig/branches/spark/build.xml (original)
+++ pig/branches/spark/build.xml Wed Apr 22 01:22:38 2015
@@ -104,8 +104,9 @@
<property name="test.smoke.file" value="${test.src.dir}/smoke-tests"/>
<property name="test.all.file" value="${test.src.dir}/all-tests"/>
<property name="test.tez.file" value="${test.src.dir}/tez-tests"/>
- <property name="test.spark.file" value="${test.src.dir}/spark-tests"/>
<property name="test.tez_local.file"
value="${test.src.dir}/tez-local-tests"/>
+ <property name="test.spark.file" value="${test.src.dir}/spark-tests"/>
+ <property name="test.spark_local.file"
value="${test.src.dir}/spark-local-tests"/>
<property name="test.exclude.file" value="${test.src.dir}/excluded-tests"/>
<property name="test.exclude.file.20"
value="${test.src.dir}/excluded-tests-20"/>
<property name="test.exclude.file.23"
value="${test.src.dir}/excluded-tests-23"/>
@@ -926,16 +927,22 @@
<fail if="test-tez.failed">Tests failed!</fail>
</target>
- <target name="test-spark"
depends="setSparkEnv,setWindowsPath,setLinuxPath,compile-test,jar,debugger.check,jackson-pig-3039-test-download"
description="Run spark unit tests">
- <macro-test-runner test.file="${test.spark.file}"
tests.failed="test-spark.failed"/>
- <fail if="test-spark.failed">Tests failed!</fail>
- </target>
-
<target name="test-tez-local"
depends="setTezEnv,setWindowsPath,setLinuxPath,compile-test,jar,debugger.check,jackson-pig-3039-test-download"
description="Run tez local mode unit tests">
<macro-test-runner test.file="${test.tez_local.file}"
tests.failed="test-tez.failed"/>
<fail if="test-tez.failed">Tests failed!</fail>
</target>
+ <target name="test-spark"
depends="setSparkEnv,setWindowsPath,setLinuxPath,compile-test,jar,debugger.check,jackson-pig-3039-test-download"
description="Run Spark unit tests in Spark cluster-local mode">
+ <macro-test-runner test.file="${test.spark.file}"
tests.failed="test-spark.failed"/>
+ <fail if="test-spark.failed">Tests failed!</fail>
+ </target>
+
+ <target name="test-spark-local"
depends="setSparkEnv,setWindowsPath,setLinuxPath,compile-test,jar,debugger.check,jackson-pig-3039-test-download"
description="Run spark unit tests in Spark local mode">
+ <macro-test-runner test.file="${test.spark_local.file}"
tests.failed="test-spark.failed"/>
+ <fail if="test-spark.failed">Tests failed!</fail>
+ </target>
+
+
<target name="debugger.check" depends="debugger.set,debugger.unset"/>
<target name="debugger.set" if="debugPort">
<property name="debugArgs" value="-Xdebug
-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=${debugPort}"/>
Modified: pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType
URL:
http://svn.apache.org/viewvc/pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType?rev=1675242&r1=1675241&r2=1675242&view=diff
==============================================================================
--- pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType (original)
+++ pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType Wed Apr 22
01:22:38 2015
@@ -16,4 +16,4 @@ org.apache.pig.backend.hadoop.executione
#org.apache.pig.backend.hadoop.executionengine.tez.TezLocalExecType
#org.apache.pig.backend.hadoop.executionengine.tez.TezExecType
org.apache.pig.backend.hadoop.executionengine.spark.SparkExecType
-
+org.apache.pig.backend.hadoop.executionengine.spark.SparkLocalExecType
Modified:
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
URL:
http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java?rev=1675242&r1=1675241&r2=1675242&view=diff
==============================================================================
---
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
(original)
+++
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
Wed Apr 22 01:22:38 2015
@@ -136,7 +136,7 @@ public class SparkLauncher extends Launc
.getExecutionEngine().instantiatePigStats();
PigStats.start(sparkStats);
- startSparkIfNeeded();
+ startSparkIfNeeded(pigContext);
// Set a unique group id for this query, so we can lookup all
Spark job
// ids
@@ -388,12 +388,17 @@ public class SparkLauncher extends Launc
return sparkPlan;
}
- private static void startSparkIfNeeded() throws PigException {
+ private static void startSparkIfNeeded(PigContext pc) throws
PigException {
if (sparkContext == null) {
- String master = System.getenv("SPARK_MASTER");
- if (master == null) {
- LOG.info("SPARK_MASTER not specified, using
\"local\"");
+ String master = null;
+ if (pc.getExecType().isLocal()) {
master = "local";
+ } else {
+ master = System.getenv("SPARK_MASTER");
+ if (master == null) {
+ LOG.info("SPARK_MASTER not specified,
using \"local\"");
+ master = "local";
+ }
}
String sparkHome = System.getenv("SPARK_HOME");
Added:
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLocalExecType.java
URL:
http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLocalExecType.java?rev=1675242&view=auto
==============================================================================
---
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLocalExecType.java
(added)
+++
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLocalExecType.java
Wed Apr 22 01:22:38 2015
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.backend.hadoop.executionengine.spark;
+
+import java.util.Properties;
+
+/**
+ * SparkLocalExecType is the ExecType for local mode in Spark.
+ */
+public class SparkLocalExecType extends SparkExecType {
+
+ private static final long serialVersionUID = 1L;
+ private static final String mode ="SPARK_LOCAL";
+
+ @Override
+ public boolean accepts(Properties properties) {
+ String execTypeSpecified = properties.getProperty("exectype", "")
+ .toUpperCase();
+ if (execTypeSpecified.equals(mode))
+ return true;
+ return false;
+ }
+
+ @Override
+ public boolean isLocal() {
+ return true;
+ }
+
+ @Override
+ public String name() {
+ return "SPARK_LOCAL";
+ }
+}
Modified: pig/branches/spark/test/org/apache/pig/test/Util.java
URL:
http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/Util.java?rev=1675242&r1=1675241&r2=1675242&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/Util.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/Util.java Wed Apr 22 01:22:38
2015
@@ -1350,11 +1350,15 @@ public class Util {
public static ExecType getLocalTestMode() throws Exception {
String execType = System.getProperty("test.exec.type");
- if (execType!=null && execType.equals("tez")) {
- return ExecTypeProvider.fromString("tez_local");
- } else {
- return ExecTypeProvider.fromString("local");
+ if (execType != null) {
+ if (execType.equals("tez")) {
+ return ExecTypeProvider.fromString("tez_local");
+ } else if (execType.equals("spark")) {
+ return ExecTypeProvider.fromString("spark_local");
+ }
}
+
+ return ExecTypeProvider.fromString("local");
}
public static void createLogAppender(Class clazz, String appenderName,
Writer writer) {
Added: pig/branches/spark/test/spark-local-tests
URL:
http://svn.apache.org/viewvc/pig/branches/spark/test/spark-local-tests?rev=1675242&view=auto
==============================================================================
--- pig/branches/spark/test/spark-local-tests (added)
+++ pig/branches/spark/test/spark-local-tests Wed Apr 22 01:22:38 2015
@@ -0,0 +1,84 @@
+**/TestAccumuloPigCluster.java
+**/TestBigTypeSort.java
+**/TestCurrentTime.java
+**/TestDefaultDateTimeZone.java
+**/TestHBaseStorage.java
+**/TestInvokerGenerator.java
+**/TestOrcStorage.java
+**/TestStreamingUDF.java
+**/TestPluckTuple.java
+**/TestImplicitSplitOnTuple.java
+**/TestMockStorage.java
+**/TestLogicalPlanGenerator.java
+**/TestOrcStoragePushdown.java
+**/TestSchemaResetter.java
+**/TestLocationInPhysicalPlan.java
+**/TestAlgebraicEvalLocal.java
+**/TestAssert.java
+**/TestBatchAliases.java
+**/TestBlackAndWhitelistValidator.java
+**/TestBuiltin.java
+**/TestBuiltInBagToTupleOrString.java
+**/TestBuiltinInvoker.java
+**/TestCase.java
+**/TestCommit.java
+**/TestDataBagAccess.java
+**/TestFilterOpNumeric.java
+**/TestFilterOpString.java
+**/TestForEachNestedPlanLocal.java
+**/TestForEachStar.java
+**/TestIn.java
+**/TestInfixArithmetic.java
+**/TestInputOutputFileValidator.java
+**/TestJsonLoaderStorage.java
+**/TestLimitSchemaStore.java
+**/TestLoad.java
+**/TestLocal.java
+**/TestLocal2.java
+**/TestLOLoadDeterminedSchema.java
+**/TestMacroExpansion.java
+**/TestNullConstant.java
+**/TestOrderBy3.java
+**/TestParser.java
+**/TestPigContext.java
+**/TestPigSplit.java
+**/TestPigStorage.java
+**/TestPoissonSampleLoader.java
+**/TestPONegative.java
+**/TestProject.java
+**/TestProjectStarExpander.java
+**/TestRank1.java
+**/TestRank2.java
+**/TestRegisteredJarVisibility.java
+**/TestRelationToExprProject.java
+**/TestScriptingLanguagePython.java
+**/TestSplit.java
+**/TestSplitIndex.java
+**/TestStoreInstances.java
+**/TestStreamingLocal.java
+**/TestTypedMap.java
+**/TestUDF.java
+**/TestUDFContext.java
+**/TestUDFGroovy.java
+**/TestUDFWithoutParameter.java
+**/TestUnion.java
+**/TestUTF8.java
+**/TestGruntParser.java
+**/TestAutoLocalMode.java
+**/TestEvalPipelineLocal.java
+**/TestFetch.java
+**/TestPigRunner.java
+**/TestPigServerWithMacros.java
+**/TestMultiQueryBasic.java
+**/TestLoadStoreFuncLifeCycle.java
+**/TestCubeOperator.java
+**/TestJoinLocal.java
+**/TestMultiQuery.java
+**/TestNewPlanColumnPrune.java
+**/TestPigServerLocal.java
+**/TestPruneColumn.java
+**/TestRank3.java
+**/TestScalarAliasesLocal.java
+**/TestStoreLocal.java
+**/TestMultiQueryLocal.java
+**/TestUnionOnSchema.java
