change method of testing for spark
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/ce4a4898 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/ce4a4898 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/ce4a4898 Branch: refs/heads/JIRA-22/pr-385 Commit: ce4a48980e33b9f16c74a62fcea6878f28b9c08b Parents: 8d9f0d4 Author: amaya <[email protected]> Authored: Fri Sep 30 17:05:20 2016 +0900 Committer: amaya <[email protected]> Committed: Fri Sep 30 17:05:20 2016 +0900 ---------------------------------------------------------------------- .../spark/sql/hive/HivemallOpsSuite.scala | 23 ++++++++++---------- .../spark/sql/hive/HivemallOpsSuite.scala | 17 ++++++--------- 2 files changed, 18 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/ce4a4898/spark/spark-1.6/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala ---------------------------------------------------------------------- diff --git a/spark/spark-1.6/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala b/spark/spark-1.6/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala index cce22ce..c7016c0 100644 --- a/spark/spark-1.6/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala +++ b/spark/spark-1.6/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala @@ -24,7 +24,6 @@ import org.apache.spark.sql.{Column, Row} import org.apache.spark.test.HivemallQueryTest import org.apache.spark.test.TestDoubleWrapper._ import org.apache.spark.test.TestUtils._ -import org.scalatest.Matchers._ final class HivemallOpsSuite extends HivemallQueryTest { @@ -189,7 +188,6 @@ final class HivemallOpsSuite extends HivemallQueryTest { test("ftvec.selection - chi2") { import hiveContext.implicits._ - implicit val doubleEquality = org.scalactic.TolerantNumerics.tolerantDoubleEquality(1e-5) // see also hivemall.ftvec.selection.ChiSquareUDFTest val df = Seq( @@ -204,17 +202,17 @@ final class HivemallOpsSuite extends HivemallQueryTest { .toDF("arg0", "arg1") val result = df.select(chi2(df("arg0"), df("arg1"))).collect - result should have length 1 + assert(result.length == 1) val chi2Val = result.head.getAs[Row](0).getAs[Seq[Double]](0) val pVal = result.head.getAs[Row](0).getAs[Seq[Double]](1) (chi2Val, Seq(10.81782088, 3.59449902, 116.16984746, 67.24482759)) .zipped - .foreach((actual, expected) => actual shouldEqual expected) + .foreach((actual, expected) => assert(actual ~== expected)) (pVal, Seq(4.47651499e-03, 1.65754167e-01, 5.94344354e-26, 2.50017968e-15)) .zipped - .foreach((actual, expected) => actual shouldEqual expected) + .foreach((actual, expected) => assert(actual ~== expected)) } test("ftvec.conv - quantify") { @@ -370,8 +368,9 @@ final class HivemallOpsSuite extends HivemallQueryTest { val data = Seq(Seq(0, 1, 3), Seq(2, 4, 1), Seq(5, 4, 9)) val df = data.map(d => (d, Seq(3, 1, 2), 2)).toDF("features", "importance_list", "k") - df.select(select_k_best(df("features"), df("importance_list"), df("k"))).collect shouldEqual - data.map(s => Row(Seq(s(0).toDouble, s(2).toDouble))) + // if use checkAnswer here, fail for some reason, maybe type? but it's okay on spark-2.0 + assert(df.select(select_k_best(df("features"), df("importance_list"), df("k"))).collect === + data.map(s => Row(Seq(s(0).toDouble, s(2).toDouble)))) } test("misc - sigmoid") { @@ -573,7 +572,6 @@ final class HivemallOpsSuite extends HivemallQueryTest { test("user-defined aggregators for ftvec.selection") { import hiveContext.implicits._ - implicit val doubleEquality = org.scalactic.TolerantNumerics.tolerantDoubleEquality(1e-5) // see also hivemall.ftvec.selection.SignalNoiseRatioUDAFTest // binary class @@ -595,7 +593,7 @@ final class HivemallOpsSuite extends HivemallQueryTest { val row0 = df0.groupby($"c0").snr("arg0", "arg1").collect (row0(0).getAs[Seq[Double]](1), Seq(4.38425236, 0.26390002, 15.83984511, 26.87005769)) .zipped - .foreach((actual, expected) => actual shouldEqual expected) + .foreach((actual, expected) => assert(actual ~== expected)) // multiple class // +-----------------+-------+ @@ -616,7 +614,7 @@ final class HivemallOpsSuite extends HivemallQueryTest { val row1 = df1.groupby($"c0").snr("arg0", "arg1").collect (row1(0).getAs[Seq[Double]](1), Seq(8.43181818, 1.32121212, 42.94949495, 33.80952381)) .zipped - .foreach((actual, expected) => actual shouldEqual expected) + .foreach((actual, expected) => assert(actual ~== expected)) } test("user-defined aggregators for tools.matrix") { @@ -627,7 +625,8 @@ final class HivemallOpsSuite extends HivemallQueryTest { val df0 = Seq((1, Seq(1, 2, 3), Seq(5, 6, 7)), (1, Seq(3, 4, 5), Seq(7, 8, 9))) .toDF("c0", "arg0", "arg1") - df0.groupby($"c0").transpose_and_dot("arg0", "arg1").collect() shouldEqual - Seq(Row(1, Seq(Seq(26.0, 30.0, 34.0), Seq(38.0, 44.0, 50.0), Seq(50.0, 58.0, 66.0)))) + // if use checkAnswer here, fail for some reason, maybe type? but it's okay on spark-2.0 + assert(df0.groupby($"c0").transpose_and_dot("arg0", "arg1").collect() === + Seq(Row(1, Seq(Seq(26.0, 30.0, 34.0), Seq(38.0, 44.0, 50.0), Seq(50.0, 58.0, 66.0))))) } } http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/ce4a4898/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala ---------------------------------------------------------------------- diff --git a/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala b/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala index fe73a1b..8446677 100644 --- a/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala +++ b/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala @@ -23,7 +23,6 @@ import org.apache.spark.sql.types._ import org.apache.spark.sql.{AnalysisException, Column, Row, functions} import org.apache.spark.test.TestDoubleWrapper._ import org.apache.spark.test.{HivemallFeatureQueryTest, TestUtils, VectorQueryTest} -import org.scalatest.Matchers._ final class HivemallOpsWithFeatureSuite extends HivemallFeatureQueryTest { @@ -188,7 +187,6 @@ final class HivemallOpsWithFeatureSuite extends HivemallFeatureQueryTest { test("ftvec.selection - chi2") { import hiveContext.implicits._ - implicit val doubleEquality = org.scalactic.TolerantNumerics.tolerantDoubleEquality(1e-5) // see also hivemall.ftvec.selection.ChiSquareUDFTest val df = Seq( @@ -203,17 +201,17 @@ final class HivemallOpsWithFeatureSuite extends HivemallFeatureQueryTest { .toDF("arg0", "arg1") val result = df.select(chi2(df("arg0"), df("arg1"))).collect - result should have length 1 + assert(result.length == 1) val chi2Val = result.head.getAs[Row](0).getAs[Seq[Double]](0) val pVal = result.head.getAs[Row](0).getAs[Seq[Double]](1) (chi2Val, Seq(10.81782088, 3.59449902, 116.16984746, 67.24482759)) .zipped - .foreach((actual, expected) => actual shouldEqual expected) + .foreach((actual, expected) => assert(actual ~== expected)) (pVal, Seq(4.47651499e-03, 1.65754167e-01, 5.94344354e-26, 2.50017968e-15)) .zipped - .foreach((actual, expected) => actual shouldEqual expected) + .foreach((actual, expected) => assert(actual ~== expected)) } test("ftvec.conv - quantify") { @@ -393,8 +391,8 @@ final class HivemallOpsWithFeatureSuite extends HivemallFeatureQueryTest { val data = Seq(Seq(0, 1, 3), Seq(2, 4, 1), Seq(5, 4, 9)) val df = data.map(d => (d, Seq(3, 1, 2), 2)).toDF("features", "importance_list", "k") - df.select(select_k_best(df("features"), df("importance_list"), df("k"))).collect shouldEqual - data.map(s => Row(Seq(s(0).toDouble, s(2).toDouble))) + checkAnswer(df.select(select_k_best(df("features"), df("importance_list"), df("k"))), + data.map(s => Row(Seq(s(0).toDouble, s(2).toDouble)))) } test("misc - sigmoid") { @@ -689,7 +687,6 @@ final class HivemallOpsWithFeatureSuite extends HivemallFeatureQueryTest { test("user-defined aggregators for ftvec.selection") { import hiveContext.implicits._ - implicit val doubleEquality = org.scalactic.TolerantNumerics.tolerantDoubleEquality(1e-5) // see also hivemall.ftvec.selection.SignalNoiseRatioUDAFTest // binary class @@ -711,7 +708,7 @@ final class HivemallOpsWithFeatureSuite extends HivemallFeatureQueryTest { val row0 = df0.groupby($"c0").snr("arg0", "arg1").collect (row0(0).getAs[Seq[Double]](1), Seq(4.38425236, 0.26390002, 15.83984511, 26.87005769)) .zipped - .foreach((actual, expected) => actual shouldEqual expected) + .foreach((actual, expected) => assert(actual ~== expected)) // multiple class // +-----------------+-------+ @@ -732,7 +729,7 @@ final class HivemallOpsWithFeatureSuite extends HivemallFeatureQueryTest { val row1 = df1.groupby($"c0").snr("arg0", "arg1").collect (row1(0).getAs[Seq[Double]](1), Seq(8.43181818, 1.32121212, 42.94949495, 33.80952381)) .zipped - .foreach((actual, expected) => actual shouldEqual expected) + .foreach((actual, expected) => assert(actual ~== expected)) } test("user-defined aggregators for tools.matrix") {
