Github user dongjoon-hyun commented on a diff in the pull request:
https://github.com/apache/spark/pull/22932#discussion_r232444034
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcOutputWriter.scala
---
@@ -36,11 +37,17 @@ private[orc] class OrcOutputWriter(
private[this] val serializer = new OrcSerializer(dataSchema)
private val recordWriter = {
- new OrcOutputFormat[OrcStruct]() {
+ val orcOutputFormat = new OrcOutputFormat[OrcStruct]() {
override def getDefaultWorkFile(context: TaskAttemptContext,
extension: String): Path = {
new Path(path)
}
- }.getRecordWriter(context)
+ }
+ val filename = orcOutputFormat.getDefaultWorkFile(context, ".orc")
+ val options =
OrcMapRedOutputFormat.buildOptions(context.getConfiguration)
+ val writer = OrcFile.createWriter(filename, options)
+ val recordWriter = new OrcMapreduceRecordWriter[OrcStruct](writer)
--- End diff --
Right. To avoid reflection, this was the only way.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
