Github user gatorsmile commented on a diff in the pull request:
https://github.com/apache/spark/pull/18714#discussion_r159352979
--- Diff:
core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala
---
@@ -85,11 +104,16 @@ class HadoopMapReduceCommitProtocol(jobId: String,
path: String)
taskContext: TaskAttemptContext, dir: Option[String], ext: String):
String = {
val filename = getFilename(taskContext, ext)
- val stagingDir: String = committer match {
+ val stagingDir: Path = committer match {
+ case _ if dynamicPartitionOverwrite =>
+ assert(dir.isDefined,
+ "The dataset to be written must be partitioned when
runtimeOverwritePartition is true.")
--- End diff --
`runtimeOverwritePartition` -> `dynamicPartitionOverwrite`
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
