Github user dongjoon-hyun commented on a diff in the pull request:
https://github.com/apache/spark/pull/20240#discussion_r161107654
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
---
@@ -608,4 +609,33 @@ class OrcQuerySuite extends OrcQueryTest with
SharedSQLContext {
}
}
}
+
+ test("Enabling/disabling ignoreCorruptFiles") {
+ def testIgnoreCorruptFiles(): Unit = {
+ withTempDir { dir =>
+ val basePath = dir.getCanonicalPath
+ spark.range(1).toDF("a").write.orc(new Path(basePath,
"first").toString)
+ spark.range(1, 2).toDF("a").write.orc(new Path(basePath,
"second").toString)
+ spark.range(2, 3).toDF("a").write.json(new Path(basePath,
"third").toString)
+ val df = spark.read.orc(
+ new Path(basePath, "first").toString,
+ new Path(basePath, "second").toString,
+ new Path(basePath, "third").toString)
+ checkAnswer(
+ df,
+ Seq(Row(0), Row(1)))
--- End diff --
Oh, thank you for review, @HyukjinKwon .
It comes from Parquet test suite. So, I keep the same shape so far.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
