Github user gatorsmile commented on a diff in the pull request:
https://github.com/apache/spark/pull/20959#discussion_r179934767
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
---
@@ -1279,4 +1279,45 @@ class CSVSuite extends QueryTest with
SharedSQLContext with SQLTestUtils {
Row("0,2013-111-11 12:13:14") :: Row(null) :: Nil
)
}
+
+ test("SPARK-23846: schema inferring touches less data if samplingRation
< 1.0") {
+ val predefinedSample = Set[Int](2, 8, 15, 27, 30, 34, 35, 37, 44, 46,
+ 57, 62, 68, 72)
+ withTempPath { path =>
+ val writer = Files.newBufferedWriter(Paths.get(path.getAbsolutePath),
+ StandardCharsets.UTF_8, StandardOpenOption.CREATE_NEW)
+ for (i <- 0 until 100) {
+ if (predefinedSample.contains(i)) {
+ writer.write(i.toString + "\n")
+ } else {
+ writer.write((i.toDouble + 0.1).toString + "\n")
+ }
+ }
+ writer.close()
+
+ val ds = spark.read
+ .option("inferSchema", true)
+ .option("samplingRatio", 0.1)
+ .csv(path.getCanonicalPath)
+ assert(ds.schema == new StructType().add("_c0", IntegerType))
+ }
+ }
+
+ test("SPARK-23846: usage of samplingRation while parsing of dataset of
strings") {
+ val dstr = spark.sparkContext.parallelize(0 until 100, 1).map { i =>
+ val predefinedSample = Set[Int](2, 8, 15, 27, 30, 34, 35, 37, 44, 46,
+ 57, 62, 68, 72)
+ if (predefinedSample.contains(i)) {
+ i.toString + "\n"
+ } else {
+ (i.toDouble + 0.1) + "\n"
+ }
+ }.toDS()
+ val ds = spark.read
+ .option("inferSchema", true)
+ .option("samplingRatio", 0.1)
--- End diff --
Add some negative case.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
