Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/22157#discussion_r212178321
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
---
@@ -562,20 +562,57 @@ abstract class OrcQueryTest extends OrcTest {
}
}
+ def testAllCorruptFiles(): Unit = {
+ withTempDir { dir =>
+ val basePath = dir.getCanonicalPath
+ spark.range(1).toDF("a").write.json(new Path(basePath,
"first").toString)
+ spark.range(1, 2).toDF("a").write.json(new Path(basePath,
"second").toString)
+ val df = spark.read.orc(
+ new Path(basePath, "first").toString,
+ new Path(basePath, "second").toString)
+ assert(df.count() == 0)
+ }
+ }
+
+ def testAllCorruptFilesWithoutSchemaInfer(): Unit = {
+ withTempDir { dir =>
+ val basePath = dir.getCanonicalPath
+ spark.range(1).toDF("a").write.json(new Path(basePath,
"first").toString)
+ spark.range(1, 2).toDF("a").write.json(new Path(basePath,
"second").toString)
+ val df = spark.read.schema("a long").orc(
+ new Path(basePath, "first").toString,
+ new Path(basePath, "second").toString)
+ assert(df.count() == 0)
+ }
+ }
+
withSQLConf(SQLConf.IGNORE_CORRUPT_FILES.key -> "true") {
testIgnoreCorruptFiles()
testIgnoreCorruptFilesWithoutSchemaInfer()
+ val m1 = intercept[AnalysisException] {
+ testAllCorruptFiles()
+ }.getMessage
+ assert(m1.contains("Unable to infer schema for ORC"))
+ testAllCorruptFilesWithoutSchemaInfer()
}
withSQLConf(SQLConf.IGNORE_CORRUPT_FILES.key -> "false") {
val m1 = intercept[SparkException] {
testIgnoreCorruptFiles()
}.getMessage
- assert(m1.contains("Could not read footer for file"))
+ assert(m1.contains("Malformed ORC file"))
--- End diff --
why the error message changed?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
