Github user gatorsmile commented on a diff in the pull request:

    https://github.com/apache/spark/pull/20959#discussion_r179934767
  
    --- Diff: 
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
 ---
    @@ -1279,4 +1279,45 @@ class CSVSuite extends QueryTest with 
SharedSQLContext with SQLTestUtils {
           Row("0,2013-111-11 12:13:14") :: Row(null) :: Nil
         )
       }
    +
    +  test("SPARK-23846: schema inferring touches less data if samplingRation 
< 1.0") {
    +    val predefinedSample = Set[Int](2, 8, 15, 27, 30, 34, 35, 37, 44, 46,
    +      57, 62, 68, 72)
    +    withTempPath { path =>
    +      val writer = Files.newBufferedWriter(Paths.get(path.getAbsolutePath),
    +        StandardCharsets.UTF_8, StandardOpenOption.CREATE_NEW)
    +      for (i <- 0 until 100) {
    +        if (predefinedSample.contains(i)) {
    +          writer.write(i.toString + "\n")
    +        } else {
    +          writer.write((i.toDouble + 0.1).toString + "\n")
    +        }
    +      }
    +      writer.close()
    +
    +      val ds = spark.read
    +        .option("inferSchema", true)
    +        .option("samplingRatio", 0.1)
    +        .csv(path.getCanonicalPath)
    +      assert(ds.schema == new StructType().add("_c0", IntegerType))
    +    }
    +  }
    +
    +  test("SPARK-23846: usage of samplingRation while parsing of dataset of 
strings") {
    +    val dstr = spark.sparkContext.parallelize(0 until 100, 1).map { i =>
    +      val predefinedSample = Set[Int](2, 8, 15, 27, 30, 34, 35, 37, 44, 46,
    +        57, 62, 68, 72)
    +      if (predefinedSample.contains(i)) {
    +        i.toString + "\n"
    +      } else {
    +        (i.toDouble + 0.1) + "\n"
    +      }
    +    }.toDS()
    +    val ds = spark.read
    +      .option("inferSchema", true)
    +      .option("samplingRatio", 0.1)
    --- End diff --
    
    Add some negative case. 


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

Reply via email to