Github user jiangxb1987 commented on a diff in the pull request:
https://github.com/apache/spark/pull/20331#discussion_r163354261
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonHadoopFsRelationSuite.scala
---
@@ -110,14 +113,16 @@ class JsonHadoopFsRelationSuite extends
HadoopFsRelationTest {
test("invalid json with leading nulls - from file (multiLine=true)") {
import testImplicits._
- withTempDir { tempDir =>
- val path = tempDir.getAbsolutePath
- Seq(badJson, """{"a":1}""").toDS().write.mode("overwrite").text(path)
- val expected = s"""$badJson\n{"a":1}\n"""
- val schema = new StructType().add("a",
IntegerType).add("_corrupt_record", StringType)
- val df =
- spark.read.format(dataSourceName).option("multiLine",
true).schema(schema).load(path)
- checkAnswer(df, Row(null, expected))
+ withSQLConf(SQLConf.MAX_RECORDS_PER_FILE.key -> "2") {
--- End diff --
Just curious, why this change?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
