Github user cloud-fan commented on a diff in the pull request:

    https://github.com/apache/spark/pull/20331#discussion_r173071130
  
    --- Diff: 
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonHadoopFsRelationSuite.scala
 ---
    @@ -110,14 +113,16 @@ class JsonHadoopFsRelationSuite extends 
HadoopFsRelationTest {
     
       test("invalid json with leading nulls - from file (multiLine=true)") {
         import testImplicits._
    -    withTempDir { tempDir =>
    -      val path = tempDir.getAbsolutePath
    -      Seq(badJson, """{"a":1}""").toDS().write.mode("overwrite").text(path)
    -      val expected = s"""$badJson\n{"a":1}\n"""
    -      val schema = new StructType().add("a", 
IntegerType).add("_corrupt_record", StringType)
    -      val df =
    -        spark.read.format(dataSourceName).option("multiLine", 
true).schema(schema).load(path)
    -      checkAnswer(df, Row(null, expected))
    +    withSQLConf(SQLConf.MAX_RECORDS_PER_FILE.key -> "2") {
    --- End diff --
    
    I think the default value won't be less than 2, we don't need to be so 
careful...


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

Reply via email to