andygrove commented on code in PR #5763:
URL: https://github.com/apache/datafusion-comet/pull/5763#discussion_r3999916642
##########
native/core/src/execution/operators/parquet_writer.rs:
##########
@@ -510,15 +514,18 @@ impl ExecutionPlan for ParquetWriterExec {
Arc::new(Schema::new(fields))
});
- // Generate part file name for this partition
- // If using FileCommitProtocol (work_dir is set), include
task_attempt_id in the filename
- let part_file = if let Some(attempt_id) = task_attempt_id {
- format!(
- "{}/part-{:05}-{:05}.parquet",
- work_dir, self.partition_id, attempt_id
- )
- } else {
- format!("{}/part-{:05}.parquet", work_dir, self.partition_id)
+ // Spark 4.0+ hands over the exact file to write, chosen by the JVM
commit protocol.
+ // Spark 3.x hands over a working directory instead and expects the
writer to name the
+ // file; that branch goes away with Spark 3.x support.
+ let part_file = match &work_dir {
+ None => self.output_path.clone(),
+ Some(work_dir) => match task_attempt_id {
+ Some(attempt_id) => format!(
+ "{}/part-{:05}-{:05}.parquet",
+ work_dir, self.partition_id, attempt_id
+ ),
+ None => format!("{}/part-{:05}.parquet", work_dir,
self.partition_id),
+ },
Review Comment:
Applied as suggested in 268fdb9d2.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
