This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 4cc7799 [SPARK-31560][SQL][TESTS] Add V1/V2 tests for TextSuite and
WholeTextFileSuite
4cc7799 is described below
commit 4cc779999bb708426a319314b549da6a8a0951d6
Author: Gengliang Wang <[email protected]>
AuthorDate: Fri Apr 24 18:59:15 2020 -0700
[SPARK-31560][SQL][TESTS] Add V1/V2 tests for TextSuite and
WholeTextFileSuite
### What changes were proposed in this pull request?
Add V1/V2 tests for TextSuite and WholeTextFileSuite
### Why are the changes needed?
This is missing part since #24207. We should have these tests for test
coverage.
### Does this PR introduce any user-facing change?
No
### How was this patch tested?
Unit tests.
Closes #28335 from gengliangwang/testV2Suite.
Authored-by: Gengliang Wang <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 16b961526dcd7062be3e0ce59f9cbcf22a6887ea)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../sql/execution/datasources/text/TextSuite.scala | 18 ++++++++++++++++--
.../datasources/text/WholeTextFileSuite.scala | 17 ++++++++++++++++-
2 files changed, 32 insertions(+), 3 deletions(-)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
index 539ff0d..7e97994 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
@@ -24,14 +24,14 @@ import java.nio.file.Files
import org.apache.hadoop.io.SequenceFile.CompressionType
import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.spark.TestUtils
+import org.apache.spark.{SparkConf, TestUtils}
import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row,
SaveMode}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types.{StringType, StructType}
import org.apache.spark.util.Utils
-class TextSuite extends QueryTest with SharedSparkSession {
+abstract class TextSuite extends QueryTest with SharedSparkSession {
import testImplicits._
test("reading text file") {
@@ -234,3 +234,17 @@ class TextSuite extends QueryTest with SharedSparkSession {
assert(data.length == 4)
}
}
+
+class TextV1Suite extends TextSuite {
+ override protected def sparkConf: SparkConf =
+ super
+ .sparkConf
+ .set(SQLConf.USE_V1_SOURCE_LIST, "text")
+}
+
+class TextV2Suite extends TextSuite {
+ override protected def sparkConf: SparkConf =
+ super
+ .sparkConf
+ .set(SQLConf.USE_V1_SOURCE_LIST, "")
+}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala
index 5e3b344..f481284 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala
@@ -19,12 +19,13 @@ package org.apache.spark.sql.execution.datasources.text
import java.io.File
+import org.apache.spark.SparkConf
import org.apache.spark.sql.{QueryTest, Row}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types.{StringType, StructType}
-class WholeTextFileSuite extends QueryTest with SharedSparkSession {
+abstract class WholeTextFileSuite extends QueryTest with SharedSparkSession {
// Hadoop's FileSystem caching does not use the Configuration as part of its
cache key, which
// can cause Filesystem.get(Configuration) to return a cached instance
created with a different
@@ -103,3 +104,17 @@ class WholeTextFileSuite extends QueryTest with
SharedSparkSession {
}
}
}
+
+class WholeTextFileV1Suite extends WholeTextFileSuite {
+ override protected def sparkConf: SparkConf =
+ super
+ .sparkConf
+ .set(SQLConf.USE_V1_SOURCE_LIST, "text")
+}
+
+class WholeTextFileV2Suite extends WholeTextFileSuite {
+ override protected def sparkConf: SparkConf =
+ super
+ .sparkConf
+ .set(SQLConf.USE_V1_SOURCE_LIST, "")
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
