spark git commit: [SPARK-4373][MLLIB] fix MLlib maven tests

2014-11-12 Thread joshrosen 
Repository: spark
Updated Branches:
  refs/heads/branch-1.2 675df2afd - 5f14cdeaa


[SPARK-4373][MLLIB] fix MLlib maven tests

We want to make sure there is at most one spark context inside the same jvm. 
JoshRosen

Author: Xiangrui Meng m...@databricks.com

Closes #3235 from mengxr/SPARK-4373 and squashes the following commits:

6574b69 [Xiangrui Meng] rename LocalSparkContext to MLlibTestSparkContext
913d48d [Xiangrui Meng] make sure there is at most one spark context inside the 
same jvm

(cherry picked from commit 23f5bdf06a388e08ea5a69e848f0ecd5165aa481)
Signed-off-by: Josh Rosen joshro...@databricks.com


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5f14cdea
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5f14cdea
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5f14cdea

Branch: refs/heads/branch-1.2
Commit: 5f14cdeaa9bfaa05f01a9f9fe77386c46f511805
Parents: 675df2a
Author: Xiangrui Meng m...@databricks.com
Authored: Wed Nov 12 18:15:14 2014 -0800
Committer: Josh Rosen joshro...@databricks.com
Committed: Wed Nov 12 18:15:36 2014 -0800

--
 .../LogisticRegressionSuite.scala   | 22 +++---
 .../spark/ml/tuning/CrossValidatorSuite.scala   | 15 ---
 .../LogisticRegressionSuite.scala   |  4 +-
 .../mllib/classification/NaiveBayesSuite.scala  |  4 +-
 .../spark/mllib/classification/SVMSuite.scala   |  4 +-
 .../spark/mllib/clustering/KMeansSuite.scala|  4 +-
 .../mllib/evaluation/AreaUnderCurveSuite.scala  |  4 +-
 .../BinaryClassificationMetricsSuite.scala  |  4 +-
 .../evaluation/MulticlassMetricsSuite.scala |  4 +-
 .../evaluation/MultilabelMetricsSuite.scala |  4 +-
 .../mllib/evaluation/RankingMetricsSuite.scala  |  4 +-
 .../evaluation/RegressionMetricsSuite.scala |  4 +-
 .../spark/mllib/feature/HashingTFSuite.scala|  4 +-
 .../apache/spark/mllib/feature/IDFSuite.scala   |  4 +-
 .../spark/mllib/feature/NormalizerSuite.scala   |  4 +-
 .../mllib/feature/StandardScalerSuite.scala |  4 +-
 .../spark/mllib/feature/Word2VecSuite.scala |  4 +-
 .../distributed/CoordinateMatrixSuite.scala |  4 +-
 .../distributed/IndexedRowMatrixSuite.scala |  4 +-
 .../linalg/distributed/RowMatrixSuite.scala |  4 +-
 .../optimization/GradientDescentSuite.scala |  4 +-
 .../spark/mllib/optimization/LBFGSSuite.scala   |  4 +-
 .../spark/mllib/random/RandomRDDsSuite.scala|  4 +-
 .../spark/mllib/rdd/RDDFunctionsSuite.scala |  4 +-
 .../spark/mllib/recommendation/ALSSuite.scala   |  4 +-
 .../spark/mllib/regression/LassoSuite.scala |  4 +-
 .../regression/LinearRegressionSuite.scala  |  4 +-
 .../mllib/regression/RidgeRegressionSuite.scala |  4 +-
 .../spark/mllib/stat/CorrelationSuite.scala |  4 +-
 .../spark/mllib/stat/HypothesisTestSuite.scala  |  4 +-
 .../spark/mllib/tree/DecisionTreeSuite.scala|  4 +-
 .../mllib/tree/GradientBoostingSuite.scala  |  4 +-
 .../spark/mllib/tree/RandomForestSuite.scala|  4 +-
 .../mllib/tree/impl/BaggedPointSuite.scala  |  4 +-
 .../spark/mllib/util/LocalSparkContext.scala| 33 ---
 .../apache/spark/mllib/util/MLUtilsSuite.scala  |  2 +-
 .../mllib/util/MLlibTestSparkContext.scala  | 42 
 37 files changed, 134 insertions(+), 108 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/5f14cdea/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
--
diff --git 
a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
 
b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
index 625af29..e8030fe 100644
--- 
a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
+++ 
b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
@@ -20,16 +20,24 @@ package org.apache.spark.ml.classification
 import org.scalatest.FunSuite
 
 import 
org.apache.spark.mllib.classification.LogisticRegressionSuite.generateLogisticInput
-import org.apache.spark.mllib.util.LocalSparkContext
-import org.apache.spark.sql.SchemaRDD
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+import org.apache.spark.sql.{SQLContext, SchemaRDD}
 
-class LogisticRegressionSuite extends FunSuite with LocalSparkContext {
+class LogisticRegressionSuite extends FunSuite with MLlibTestSparkContext {
 
-  import sqlContext._
+  @transient var sqlContext: SQLContext = _
+  @transient var dataset: SchemaRDD = _
 
-  val dataset: SchemaRDD = sc.parallelize(generateLogisticInput(1.0, 1.0, 100, 
42), 2)
+  override def beforeAll(): Unit = {
+super.beforeAll()
+sqlContext = new SQLContext(sc)
+dataset =

spark git commit: [SPARK-4373][MLLIB] fix MLlib maven tests

2014-11-12 Thread joshrosen 
Repository: spark
Updated Branches:
  refs/heads/master 723a86b04 - 23f5bdf06


[SPARK-4373][MLLIB] fix MLlib maven tests

We want to make sure there is at most one spark context inside the same jvm. 
JoshRosen

Author: Xiangrui Meng m...@databricks.com

Closes #3235 from mengxr/SPARK-4373 and squashes the following commits:

6574b69 [Xiangrui Meng] rename LocalSparkContext to MLlibTestSparkContext
913d48d [Xiangrui Meng] make sure there is at most one spark context inside the 
same jvm


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/23f5bdf0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/23f5bdf0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/23f5bdf0

Branch: refs/heads/master
Commit: 23f5bdf06a388e08ea5a69e848f0ecd5165aa481
Parents: 723a86b
Author: Xiangrui Meng m...@databricks.com
Authored: Wed Nov 12 18:15:14 2014 -0800
Committer: Josh Rosen joshro...@databricks.com
Committed: Wed Nov 12 18:15:14 2014 -0800

--
 .../LogisticRegressionSuite.scala   | 22 +++---
 .../spark/ml/tuning/CrossValidatorSuite.scala   | 15 ---
 .../LogisticRegressionSuite.scala   |  4 +-
 .../mllib/classification/NaiveBayesSuite.scala  |  4 +-
 .../spark/mllib/classification/SVMSuite.scala   |  4 +-
 .../spark/mllib/clustering/KMeansSuite.scala|  4 +-
 .../mllib/evaluation/AreaUnderCurveSuite.scala  |  4 +-
 .../BinaryClassificationMetricsSuite.scala  |  4 +-
 .../evaluation/MulticlassMetricsSuite.scala |  4 +-
 .../evaluation/MultilabelMetricsSuite.scala |  4 +-
 .../mllib/evaluation/RankingMetricsSuite.scala  |  4 +-
 .../evaluation/RegressionMetricsSuite.scala |  4 +-
 .../spark/mllib/feature/HashingTFSuite.scala|  4 +-
 .../apache/spark/mllib/feature/IDFSuite.scala   |  4 +-
 .../spark/mllib/feature/NormalizerSuite.scala   |  4 +-
 .../mllib/feature/StandardScalerSuite.scala |  4 +-
 .../spark/mllib/feature/Word2VecSuite.scala |  4 +-
 .../distributed/CoordinateMatrixSuite.scala |  4 +-
 .../distributed/IndexedRowMatrixSuite.scala |  4 +-
 .../linalg/distributed/RowMatrixSuite.scala |  4 +-
 .../optimization/GradientDescentSuite.scala |  4 +-
 .../spark/mllib/optimization/LBFGSSuite.scala   |  4 +-
 .../spark/mllib/random/RandomRDDsSuite.scala|  4 +-
 .../spark/mllib/rdd/RDDFunctionsSuite.scala |  4 +-
 .../spark/mllib/recommendation/ALSSuite.scala   |  4 +-
 .../spark/mllib/regression/LassoSuite.scala |  4 +-
 .../regression/LinearRegressionSuite.scala  |  4 +-
 .../mllib/regression/RidgeRegressionSuite.scala |  4 +-
 .../spark/mllib/stat/CorrelationSuite.scala |  4 +-
 .../spark/mllib/stat/HypothesisTestSuite.scala  |  4 +-
 .../spark/mllib/tree/DecisionTreeSuite.scala|  4 +-
 .../mllib/tree/GradientBoostingSuite.scala  |  4 +-
 .../spark/mllib/tree/RandomForestSuite.scala|  4 +-
 .../mllib/tree/impl/BaggedPointSuite.scala  |  4 +-
 .../spark/mllib/util/LocalSparkContext.scala| 33 ---
 .../apache/spark/mllib/util/MLUtilsSuite.scala  |  2 +-
 .../mllib/util/MLlibTestSparkContext.scala  | 42 
 37 files changed, 134 insertions(+), 108 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/23f5bdf0/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
--
diff --git 
a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
 
b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
index 625af29..e8030fe 100644
--- 
a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
+++ 
b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
@@ -20,16 +20,24 @@ package org.apache.spark.ml.classification
 import org.scalatest.FunSuite
 
 import 
org.apache.spark.mllib.classification.LogisticRegressionSuite.generateLogisticInput
-import org.apache.spark.mllib.util.LocalSparkContext
-import org.apache.spark.sql.SchemaRDD
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+import org.apache.spark.sql.{SQLContext, SchemaRDD}
 
-class LogisticRegressionSuite extends FunSuite with LocalSparkContext {
+class LogisticRegressionSuite extends FunSuite with MLlibTestSparkContext {
 
-  import sqlContext._
+  @transient var sqlContext: SQLContext = _
+  @transient var dataset: SchemaRDD = _
 
-  val dataset: SchemaRDD = sc.parallelize(generateLogisticInput(1.0, 1.0, 100, 
42), 2)
+  override def beforeAll(): Unit = {
+super.beforeAll()
+sqlContext = new SQLContext(sc)
+dataset = sqlContext.createSchemaRDD(
+  sc.parallelize(generateLogisticInput(1.0, 1.0, 100, 42), 2))
+  }
 
   test(logistic regression) {
+val