Maxim Gekk created SPARK-30509:
----------------------------------
Summary: Deprecation log warning is not printed in Avro schema
inferring
Key: SPARK-30509
URL: https://issues.apache.org/jira/browse/SPARK-30509
Project: Spark
Issue Type: Bug
Components: SQL
Affects Versions: 3.0.0
Reporter: Maxim Gekk
The bug can be reproduced by the test:
{code}
test("log a warning of ignoreExtension deprecation") {
val logAppender = new LogAppender
withTempPath { dir =>
Seq(("a", 1, 2), ("b", 1, 2), ("c", 2, 1), ("d", 2, 1))
.toDF("value", "p1", "p2")
.repartition(2)
.write
.format("avro")
.option("header", true)
.save(dir.getCanonicalPath)
withLogAppender(logAppender) {
spark
.read
.format("avro")
.option(AvroOptions.ignoreExtensionKey, false)
.option("header", true)
.load(dir.getCanonicalPath)
.count()
}
val deprecatedEvents = logAppender.loggingEvents
.filter(_.getRenderedMessage.contains(
s"Option ${AvroOptions.ignoreExtensionKey} is deprecated"))
assert(deprecatedEvents.size === 1)
}
}
{code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
