Github user nkronenfeld commented on a diff in the pull request:
https://github.com/apache/spark/pull/19529#discussion_r146311180
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala ---
@@ -52,249 +36,23 @@ import org.apache.spark.util.{UninterruptibleThread,
Utils}
* Subclasses should *not* create `SQLContext`s in the test suite
constructor, which is
* prone to leaving multiple overlapping
[[org.apache.spark.SparkContext]]s in the same JVM.
*/
-private[sql] trait SQLTestUtils
- extends SparkFunSuite with Eventually
- with BeforeAndAfterAll
- with SQLTestData
- with PlanTest { self =>
-
- protected def sparkContext = spark.sparkContext
-
+private[sql] trait SQLTestUtils extends SparkFunSuite with
SQLTestUtilsBase with PlanTest {
--- End diff --
This has more changes - I left the data initialization stuff in
SQLTestUtils, as it wouldn't be needed by external tests. All the <'s below
are things that I pulled out of SQLTestUtilsBase and put back into SQLTestUtils
Running similarly:
git checkout 4a779bdac3e75c17b7d36c5a009ba6c948fa9fb6 SQLTestUtils.scala
diff SQLTestUtils.scala SQLTestUtilsBase.scala
I get
27d26
< import scala.util.control.NonFatal
30c29
< import org.scalatest.BeforeAndAfterAll
---
> import org.scalatest.{BeforeAndAfterAll, Suite}
33d31
< import org.apache.spark.SparkFunSuite
38c36
< import org.apache.spark.sql.catalyst.plans.PlanTest
---
> import org.apache.spark.sql.catalyst.plans.PlanTestBase
40d37
< import org.apache.spark.sql.catalyst.util._
43c40
< import org.apache.spark.util.{UninterruptibleThread, Utils}
---
> import org.apache.spark.util.Utils
46c43
< * Helper trait that should be extended by all SQL test suites.
---
> * Helper trait that can be extended by all external SQL test suites.
48,49c45
< * This allows subclasses to plugin a custom `SQLContext`. It comes
with test data
< * prepared in advance as well as all implicit conversions used
extensively by dataframes.
---
> * This allows subclasses to plugin a custom `SQLContext`.
55,56c51,52
< private[sql] trait SQLTestUtils
< extends SparkFunSuite with Eventually
---
> private[sql] trait SQLTestUtilsBase
> extends Eventually
59c55
< with PlanTest { self =>
---
> with PlanTestBase { self: Suite =>
63,65d58
< // Whether to materialize all test data before the first test is run
< private var loadTestDataBeforeTests = false
<
80,94d72
< /**
< * Materialize the test data immediately after the `SQLContext` is
set up.
< * This is necessary if the data is accessed by name but not
through direct reference.
< */
< protected def setupTestData(): Unit = {
< loadTestDataBeforeTests = true
< }
<
< protected override def beforeAll(): Unit = {
< super.beforeAll()
< if (loadTestDataBeforeTests) {
< loadTestData()
< }
< }
<
300,354d277
< /**
< * Disable stdout and stderr when running the test. To not output
the logs to the console,
< * ConsoleAppender's `follow` should be set to `true` so that it
will honors reassignments of
< * System.out or System.err. Otherwise, ConsoleAppender will still
output to the console even if
< * we change System.out and System.err.
< */
< protected def testQuietly(name: String)(f: => Unit): Unit = {
< test(name) {
< quietly {
< f
< }
< }
< }
<
< /**
< * Run a test on a separate `UninterruptibleThread`.
< */
< protected def testWithUninterruptibleThread(name: String, quietly:
Boolean = false)
< (body: => Unit): Unit = {
< val timeoutMillis = 10000
< @transient var ex: Throwable = null
<
< def runOnThread(): Unit = {
< val thread = new UninterruptibleThread(s"Testing thread for
test $name") {
< override def run(): Unit = {
< try {
< body
< } catch {
< case NonFatal(e) =>
< ex = e
< }
< }
< }
< thread.setDaemon(true)
< thread.start()
< thread.join(timeoutMillis)
< if (thread.isAlive) {
< thread.interrupt()
< // If this interrupt does not work, then this thread is most
likely running something that
< // is not interruptible. There is not much point to wait for
the thread to termniate, and
< // we rather let the JVM terminate the thread on exit.
< fail(
< s"Test '$name' running on o.a.s.util.UninterruptibleThread
timed out after" +
< s" $timeoutMillis ms")
< } else if (ex != null) {
< throw ex
< }
< }
<
< if (quietly) {
< testQuietly(name) { runOnThread() }
< } else {
< test(name) { runOnThread() }
< }
< }
365,407d287
< }
< }
<
< private[sql] object SQLTestUtils {
<
< def compareAnswers(
< sparkAnswer: Seq[Row],
< expectedAnswer: Seq[Row],
< sort: Boolean): Option[String] = {
< def prepareAnswer(answer: Seq[Row]): Seq[Row] = {
< // Converts data to types that we can do equality comparison
using Scala collections.
< // For BigDecimal type, the Scala type has a better definition
of equality test (similar to
< // Java's java.math.BigDecimal.compareTo).
< // For binary arrays, we convert it to Seq to avoid of calling
java.util.Arrays.equals for
< // equality test.
< // This function is copied from Catalyst's QueryTest
< val converted: Seq[Row] = answer.map { s =>
< Row.fromSeq(s.toSeq.map {
< case d: java.math.BigDecimal => BigDecimal(d)
< case b: Array[Byte] => b.toSeq
< case o => o
< })
< }
< if (sort) {
< converted.sortBy(_.toString())
< } else {
< converted
< }
< }
< if (prepareAnswer(expectedAnswer) != prepareAnswer(sparkAnswer)) {
< val errorMessage =
< s"""
< | == Results ==
< | ${sideBySide(
< s"== Expected Answer - ${expectedAnswer.size} ==" +:
< prepareAnswer(expectedAnswer).map(_.toString()),
< s"== Actual Answer - ${sparkAnswer.size} ==" +:
<
prepareAnswer(sparkAnswer).map(_.toString())).mkString("\n")}
< """.stripMargin
< Some(errorMessage)
< } else {
< None
< }
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
