[07/10] spark git commit: [SPARK-15037][SQL][MLLIB] Use SparkSession instead of SQLContext in Scala/Java TestSuites

2016-05-10 Thread andrewor14
http://git-wip-us.apache.org/repos/asf/spark/blob/ed0b4070/mllib/src/test/scala/org/apache/spark/ml/feature/StopWordsRemoverSuite.scala
--
diff --git 
a/mllib/src/test/scala/org/apache/spark/ml/feature/StopWordsRemoverSuite.scala 
b/mllib/src/test/scala/org/apache/spark/ml/feature/StopWordsRemoverSuite.scala
index 8e7e000..125ad02 100755
--- 
a/mllib/src/test/scala/org/apache/spark/ml/feature/StopWordsRemoverSuite.scala
+++ 
b/mllib/src/test/scala/org/apache/spark/ml/feature/StopWordsRemoverSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.ml.feature
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.ml.util.DefaultReadWriteTest
 import org.apache.spark.mllib.util.MLlibTestSparkContext
-import org.apache.spark.sql.{DataFrame, Dataset, Row}
+import org.apache.spark.sql.{Dataset, Row}
 
 object StopWordsRemoverSuite extends SparkFunSuite {
   def testStopWordsRemover(t: StopWordsRemover, dataset: Dataset[_]): Unit = {
@@ -42,7 +42,7 @@ class StopWordsRemoverSuite
 val remover = new StopWordsRemover()
   .setInputCol("raw")
   .setOutputCol("filtered")
-val dataSet = sqlContext.createDataFrame(Seq(
+val dataSet = spark.createDataFrame(Seq(
   (Seq("test", "test"), Seq("test", "test")),
   (Seq("a", "b", "c", "d"), Seq("b", "c")),
   (Seq("a", "the", "an"), Seq()),
@@ -60,7 +60,7 @@ class StopWordsRemoverSuite
   .setInputCol("raw")
   .setOutputCol("filtered")
   .setStopWords(stopWords)
-val dataSet = sqlContext.createDataFrame(Seq(
+val dataSet = spark.createDataFrame(Seq(
   (Seq("test", "test"), Seq()),
   (Seq("a", "b", "c", "d"), Seq("b", "c", "d")),
   (Seq("a", "the", "an"), Seq()),
@@ -77,7 +77,7 @@ class StopWordsRemoverSuite
   .setInputCol("raw")
   .setOutputCol("filtered")
   .setCaseSensitive(true)
-val dataSet = sqlContext.createDataFrame(Seq(
+val dataSet = spark.createDataFrame(Seq(
   (Seq("A"), Seq("A")),
   (Seq("The", "the"), Seq("The"))
 )).toDF("raw", "expected")
@@ -98,7 +98,7 @@ class StopWordsRemoverSuite
   .setInputCol("raw")
   .setOutputCol("filtered")
   .setStopWords(stopWords)
-val dataSet = sqlContext.createDataFrame(Seq(
+val dataSet = spark.createDataFrame(Seq(
   (Seq("acaba", "ama", "biri"), Seq()),
   (Seq("hep", "her", "scala"), Seq("scala"))
 )).toDF("raw", "expected")
@@ -112,7 +112,7 @@ class StopWordsRemoverSuite
   .setInputCol("raw")
   .setOutputCol("filtered")
   .setStopWords(stopWords.toArray)
-val dataSet = sqlContext.createDataFrame(Seq(
+val dataSet = spark.createDataFrame(Seq(
   (Seq("python", "scala", "a"), Seq("python", "scala", "a")),
   (Seq("Python", "Scala", "swift"), Seq("Python", "Scala", "swift"))
 )).toDF("raw", "expected")
@@ -126,7 +126,7 @@ class StopWordsRemoverSuite
   .setInputCol("raw")
   .setOutputCol("filtered")
   .setStopWords(stopWords.toArray)
-val dataSet = sqlContext.createDataFrame(Seq(
+val dataSet = spark.createDataFrame(Seq(
   (Seq("python", "scala", "a"), Seq()),
   (Seq("Python", "Scala", "swift"), Seq("swift"))
 )).toDF("raw", "expected")
@@ -148,7 +148,7 @@ class StopWordsRemoverSuite
 val remover = new StopWordsRemover()
   .setInputCol("raw")
   .setOutputCol(outputCol)
-val dataSet = sqlContext.createDataFrame(Seq(
+val dataSet = spark.createDataFrame(Seq(
   (Seq("The", "the", "swift"), Seq("swift"))
 )).toDF("raw", outputCol)
 

http://git-wip-us.apache.org/repos/asf/spark/blob/ed0b4070/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
--
diff --git 
a/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala 
b/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
index d0f3cdc..c221d4a 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
@@ -39,7 +39,7 @@ class StringIndexerSuite
 
   test("StringIndexer") {
 val data = sc.parallelize(Seq((0, "a"), (1, "b"), (2, "c"), (3, "a"), (4, 
"a"), (5, "c")), 2)
-val df = sqlContext.createDataFrame(data).toDF("id", "label")
+val df = spark.createDataFrame(data).toDF("id", "label")
 val indexer = new StringIndexer()
   .setInputCol("label")
   .setOutputCol("labelIndex")
@@ -63,8 +63,8 @@ class StringIndexerSuite
   test("StringIndexerUnseen") {
 val data = sc.parallelize(Seq((0, "a"), (1, "b"), (4, "b")), 2)
 val data2 = sc.parallelize(Seq((0, "a"), (1, "b"), (2, "c")), 2)
-val df = sqlContext.createDataFrame(data).toDF("id", "label")
-val df2 = sqlContext.createDataFrame(data2).toDF("id", "label")
+val df = spark.createDataFrame(data).toDF("id", "label")
+val df2 = spark.createDataFrame(

[07/10] spark git commit: [SPARK-15037][SQL][MLLIB] Use SparkSession instead of SQLContext in Scala/Java TestSuites

2016-05-10 Thread andrewor14
http://git-wip-us.apache.org/repos/asf/spark/blob/5bf74b44/mllib/src/test/scala/org/apache/spark/ml/feature/StopWordsRemoverSuite.scala
--
diff --git 
a/mllib/src/test/scala/org/apache/spark/ml/feature/StopWordsRemoverSuite.scala 
b/mllib/src/test/scala/org/apache/spark/ml/feature/StopWordsRemoverSuite.scala
index 8e7e000..125ad02 100755
--- 
a/mllib/src/test/scala/org/apache/spark/ml/feature/StopWordsRemoverSuite.scala
+++ 
b/mllib/src/test/scala/org/apache/spark/ml/feature/StopWordsRemoverSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.ml.feature
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.ml.util.DefaultReadWriteTest
 import org.apache.spark.mllib.util.MLlibTestSparkContext
-import org.apache.spark.sql.{DataFrame, Dataset, Row}
+import org.apache.spark.sql.{Dataset, Row}
 
 object StopWordsRemoverSuite extends SparkFunSuite {
   def testStopWordsRemover(t: StopWordsRemover, dataset: Dataset[_]): Unit = {
@@ -42,7 +42,7 @@ class StopWordsRemoverSuite
 val remover = new StopWordsRemover()
   .setInputCol("raw")
   .setOutputCol("filtered")
-val dataSet = sqlContext.createDataFrame(Seq(
+val dataSet = spark.createDataFrame(Seq(
   (Seq("test", "test"), Seq("test", "test")),
   (Seq("a", "b", "c", "d"), Seq("b", "c")),
   (Seq("a", "the", "an"), Seq()),
@@ -60,7 +60,7 @@ class StopWordsRemoverSuite
   .setInputCol("raw")
   .setOutputCol("filtered")
   .setStopWords(stopWords)
-val dataSet = sqlContext.createDataFrame(Seq(
+val dataSet = spark.createDataFrame(Seq(
   (Seq("test", "test"), Seq()),
   (Seq("a", "b", "c", "d"), Seq("b", "c", "d")),
   (Seq("a", "the", "an"), Seq()),
@@ -77,7 +77,7 @@ class StopWordsRemoverSuite
   .setInputCol("raw")
   .setOutputCol("filtered")
   .setCaseSensitive(true)
-val dataSet = sqlContext.createDataFrame(Seq(
+val dataSet = spark.createDataFrame(Seq(
   (Seq("A"), Seq("A")),
   (Seq("The", "the"), Seq("The"))
 )).toDF("raw", "expected")
@@ -98,7 +98,7 @@ class StopWordsRemoverSuite
   .setInputCol("raw")
   .setOutputCol("filtered")
   .setStopWords(stopWords)
-val dataSet = sqlContext.createDataFrame(Seq(
+val dataSet = spark.createDataFrame(Seq(
   (Seq("acaba", "ama", "biri"), Seq()),
   (Seq("hep", "her", "scala"), Seq("scala"))
 )).toDF("raw", "expected")
@@ -112,7 +112,7 @@ class StopWordsRemoverSuite
   .setInputCol("raw")
   .setOutputCol("filtered")
   .setStopWords(stopWords.toArray)
-val dataSet = sqlContext.createDataFrame(Seq(
+val dataSet = spark.createDataFrame(Seq(
   (Seq("python", "scala", "a"), Seq("python", "scala", "a")),
   (Seq("Python", "Scala", "swift"), Seq("Python", "Scala", "swift"))
 )).toDF("raw", "expected")
@@ -126,7 +126,7 @@ class StopWordsRemoverSuite
   .setInputCol("raw")
   .setOutputCol("filtered")
   .setStopWords(stopWords.toArray)
-val dataSet = sqlContext.createDataFrame(Seq(
+val dataSet = spark.createDataFrame(Seq(
   (Seq("python", "scala", "a"), Seq()),
   (Seq("Python", "Scala", "swift"), Seq("swift"))
 )).toDF("raw", "expected")
@@ -148,7 +148,7 @@ class StopWordsRemoverSuite
 val remover = new StopWordsRemover()
   .setInputCol("raw")
   .setOutputCol(outputCol)
-val dataSet = sqlContext.createDataFrame(Seq(
+val dataSet = spark.createDataFrame(Seq(
   (Seq("The", "the", "swift"), Seq("swift"))
 )).toDF("raw", outputCol)
 

http://git-wip-us.apache.org/repos/asf/spark/blob/5bf74b44/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
--
diff --git 
a/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala 
b/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
index d0f3cdc..c221d4a 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
@@ -39,7 +39,7 @@ class StringIndexerSuite
 
   test("StringIndexer") {
 val data = sc.parallelize(Seq((0, "a"), (1, "b"), (2, "c"), (3, "a"), (4, 
"a"), (5, "c")), 2)
-val df = sqlContext.createDataFrame(data).toDF("id", "label")
+val df = spark.createDataFrame(data).toDF("id", "label")
 val indexer = new StringIndexer()
   .setInputCol("label")
   .setOutputCol("labelIndex")
@@ -63,8 +63,8 @@ class StringIndexerSuite
   test("StringIndexerUnseen") {
 val data = sc.parallelize(Seq((0, "a"), (1, "b"), (4, "b")), 2)
 val data2 = sc.parallelize(Seq((0, "a"), (1, "b"), (2, "c")), 2)
-val df = sqlContext.createDataFrame(data).toDF("id", "label")
-val df2 = sqlContext.createDataFrame(data2).toDF("id", "label")
+val df = spark.createDataFrame(data).toDF("id", "label")
+val df2 = spark.createDataFrame(