spark git commit: [SPARK-4373][MLLIB] fix MLlib maven tests
Repository: spark Updated Branches: refs/heads/branch-1.2 675df2afd - 5f14cdeaa [SPARK-4373][MLLIB] fix MLlib maven tests We want to make sure there is at most one spark context inside the same jvm. JoshRosen Author: Xiangrui Meng m...@databricks.com Closes #3235 from mengxr/SPARK-4373 and squashes the following commits: 6574b69 [Xiangrui Meng] rename LocalSparkContext to MLlibTestSparkContext 913d48d [Xiangrui Meng] make sure there is at most one spark context inside the same jvm (cherry picked from commit 23f5bdf06a388e08ea5a69e848f0ecd5165aa481) Signed-off-by: Josh Rosen joshro...@databricks.com Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5f14cdea Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5f14cdea Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5f14cdea Branch: refs/heads/branch-1.2 Commit: 5f14cdeaa9bfaa05f01a9f9fe77386c46f511805 Parents: 675df2a Author: Xiangrui Meng m...@databricks.com Authored: Wed Nov 12 18:15:14 2014 -0800 Committer: Josh Rosen joshro...@databricks.com Committed: Wed Nov 12 18:15:36 2014 -0800 -- .../LogisticRegressionSuite.scala | 22 +++--- .../spark/ml/tuning/CrossValidatorSuite.scala | 15 --- .../LogisticRegressionSuite.scala | 4 +- .../mllib/classification/NaiveBayesSuite.scala | 4 +- .../spark/mllib/classification/SVMSuite.scala | 4 +- .../spark/mllib/clustering/KMeansSuite.scala| 4 +- .../mllib/evaluation/AreaUnderCurveSuite.scala | 4 +- .../BinaryClassificationMetricsSuite.scala | 4 +- .../evaluation/MulticlassMetricsSuite.scala | 4 +- .../evaluation/MultilabelMetricsSuite.scala | 4 +- .../mllib/evaluation/RankingMetricsSuite.scala | 4 +- .../evaluation/RegressionMetricsSuite.scala | 4 +- .../spark/mllib/feature/HashingTFSuite.scala| 4 +- .../apache/spark/mllib/feature/IDFSuite.scala | 4 +- .../spark/mllib/feature/NormalizerSuite.scala | 4 +- .../mllib/feature/StandardScalerSuite.scala | 4 +- .../spark/mllib/feature/Word2VecSuite.scala | 4 +- .../distributed/CoordinateMatrixSuite.scala | 4 +- .../distributed/IndexedRowMatrixSuite.scala | 4 +- .../linalg/distributed/RowMatrixSuite.scala | 4 +- .../optimization/GradientDescentSuite.scala | 4 +- .../spark/mllib/optimization/LBFGSSuite.scala | 4 +- .../spark/mllib/random/RandomRDDsSuite.scala| 4 +- .../spark/mllib/rdd/RDDFunctionsSuite.scala | 4 +- .../spark/mllib/recommendation/ALSSuite.scala | 4 +- .../spark/mllib/regression/LassoSuite.scala | 4 +- .../regression/LinearRegressionSuite.scala | 4 +- .../mllib/regression/RidgeRegressionSuite.scala | 4 +- .../spark/mllib/stat/CorrelationSuite.scala | 4 +- .../spark/mllib/stat/HypothesisTestSuite.scala | 4 +- .../spark/mllib/tree/DecisionTreeSuite.scala| 4 +- .../mllib/tree/GradientBoostingSuite.scala | 4 +- .../spark/mllib/tree/RandomForestSuite.scala| 4 +- .../mllib/tree/impl/BaggedPointSuite.scala | 4 +- .../spark/mllib/util/LocalSparkContext.scala| 33 --- .../apache/spark/mllib/util/MLUtilsSuite.scala | 2 +- .../mllib/util/MLlibTestSparkContext.scala | 42 37 files changed, 134 insertions(+), 108 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/5f14cdea/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala -- diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala index 625af29..e8030fe 100644 --- a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala @@ -20,16 +20,24 @@ package org.apache.spark.ml.classification import org.scalatest.FunSuite import org.apache.spark.mllib.classification.LogisticRegressionSuite.generateLogisticInput -import org.apache.spark.mllib.util.LocalSparkContext -import org.apache.spark.sql.SchemaRDD +import org.apache.spark.mllib.util.MLlibTestSparkContext +import org.apache.spark.sql.{SQLContext, SchemaRDD} -class LogisticRegressionSuite extends FunSuite with LocalSparkContext { +class LogisticRegressionSuite extends FunSuite with MLlibTestSparkContext { - import sqlContext._ + @transient var sqlContext: SQLContext = _ + @transient var dataset: SchemaRDD = _ - val dataset: SchemaRDD = sc.parallelize(generateLogisticInput(1.0, 1.0, 100, 42), 2) + override def beforeAll(): Unit = { +super.beforeAll() +sqlContext = new SQLContext(sc) +dataset =
spark git commit: [SPARK-4373][MLLIB] fix MLlib maven tests
Repository: spark Updated Branches: refs/heads/master 723a86b04 - 23f5bdf06 [SPARK-4373][MLLIB] fix MLlib maven tests We want to make sure there is at most one spark context inside the same jvm. JoshRosen Author: Xiangrui Meng m...@databricks.com Closes #3235 from mengxr/SPARK-4373 and squashes the following commits: 6574b69 [Xiangrui Meng] rename LocalSparkContext to MLlibTestSparkContext 913d48d [Xiangrui Meng] make sure there is at most one spark context inside the same jvm Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/23f5bdf0 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/23f5bdf0 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/23f5bdf0 Branch: refs/heads/master Commit: 23f5bdf06a388e08ea5a69e848f0ecd5165aa481 Parents: 723a86b Author: Xiangrui Meng m...@databricks.com Authored: Wed Nov 12 18:15:14 2014 -0800 Committer: Josh Rosen joshro...@databricks.com Committed: Wed Nov 12 18:15:14 2014 -0800 -- .../LogisticRegressionSuite.scala | 22 +++--- .../spark/ml/tuning/CrossValidatorSuite.scala | 15 --- .../LogisticRegressionSuite.scala | 4 +- .../mllib/classification/NaiveBayesSuite.scala | 4 +- .../spark/mllib/classification/SVMSuite.scala | 4 +- .../spark/mllib/clustering/KMeansSuite.scala| 4 +- .../mllib/evaluation/AreaUnderCurveSuite.scala | 4 +- .../BinaryClassificationMetricsSuite.scala | 4 +- .../evaluation/MulticlassMetricsSuite.scala | 4 +- .../evaluation/MultilabelMetricsSuite.scala | 4 +- .../mllib/evaluation/RankingMetricsSuite.scala | 4 +- .../evaluation/RegressionMetricsSuite.scala | 4 +- .../spark/mllib/feature/HashingTFSuite.scala| 4 +- .../apache/spark/mllib/feature/IDFSuite.scala | 4 +- .../spark/mllib/feature/NormalizerSuite.scala | 4 +- .../mllib/feature/StandardScalerSuite.scala | 4 +- .../spark/mllib/feature/Word2VecSuite.scala | 4 +- .../distributed/CoordinateMatrixSuite.scala | 4 +- .../distributed/IndexedRowMatrixSuite.scala | 4 +- .../linalg/distributed/RowMatrixSuite.scala | 4 +- .../optimization/GradientDescentSuite.scala | 4 +- .../spark/mllib/optimization/LBFGSSuite.scala | 4 +- .../spark/mllib/random/RandomRDDsSuite.scala| 4 +- .../spark/mllib/rdd/RDDFunctionsSuite.scala | 4 +- .../spark/mllib/recommendation/ALSSuite.scala | 4 +- .../spark/mllib/regression/LassoSuite.scala | 4 +- .../regression/LinearRegressionSuite.scala | 4 +- .../mllib/regression/RidgeRegressionSuite.scala | 4 +- .../spark/mllib/stat/CorrelationSuite.scala | 4 +- .../spark/mllib/stat/HypothesisTestSuite.scala | 4 +- .../spark/mllib/tree/DecisionTreeSuite.scala| 4 +- .../mllib/tree/GradientBoostingSuite.scala | 4 +- .../spark/mllib/tree/RandomForestSuite.scala| 4 +- .../mllib/tree/impl/BaggedPointSuite.scala | 4 +- .../spark/mllib/util/LocalSparkContext.scala| 33 --- .../apache/spark/mllib/util/MLUtilsSuite.scala | 2 +- .../mllib/util/MLlibTestSparkContext.scala | 42 37 files changed, 134 insertions(+), 108 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/23f5bdf0/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala -- diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala index 625af29..e8030fe 100644 --- a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala @@ -20,16 +20,24 @@ package org.apache.spark.ml.classification import org.scalatest.FunSuite import org.apache.spark.mllib.classification.LogisticRegressionSuite.generateLogisticInput -import org.apache.spark.mllib.util.LocalSparkContext -import org.apache.spark.sql.SchemaRDD +import org.apache.spark.mllib.util.MLlibTestSparkContext +import org.apache.spark.sql.{SQLContext, SchemaRDD} -class LogisticRegressionSuite extends FunSuite with LocalSparkContext { +class LogisticRegressionSuite extends FunSuite with MLlibTestSparkContext { - import sqlContext._ + @transient var sqlContext: SQLContext = _ + @transient var dataset: SchemaRDD = _ - val dataset: SchemaRDD = sc.parallelize(generateLogisticInput(1.0, 1.0, 100, 42), 2) + override def beforeAll(): Unit = { +super.beforeAll() +sqlContext = new SQLContext(sc) +dataset = sqlContext.createSchemaRDD( + sc.parallelize(generateLogisticInput(1.0, 1.0, 100, 42), 2)) + } test(logistic regression) { +val