mbutrovich commented on code in PR #3858:
URL: https://github.com/apache/datafusion-comet/pull/3858#discussion_r3028911611
##########
native/shuffle/src/partitioners/multi_partition.rs:
##########
@@ -523,6 +545,69 @@ impl MultiPartitionShuffleRepartitioner {
})
}
+ /// Fast path for zero-column schemas: distribute total_rows evenly across
partitions
+ /// and emit one RecordBatch per partition carrying the row count.
+ /// Skips interleave, coalesce, and spill entirely.
+ fn shuffle_write_zero_col(
+ &self,
+ total_rows: usize,
+ start_time: Instant,
+ ) -> datafusion::common::Result<()> {
+ let num_output_partitions = self.partition_writers.len();
+ let mut offsets = vec![0u64; num_output_partitions + 1];
+
+ let output_data = OpenOptions::new()
+ .write(true)
+ .create(true)
+ .truncate(true)
+ .open(&self.output_data_file)
+ .map_err(|e| DataFusionError::Execution(format!("shuffle write
error: {e:?}")))?;
+ let mut output_data = BufWriter::with_capacity(self.write_buffer_size,
output_data);
+
+ // Distribute rows evenly: each partition gets total/N, first
(total%N) get one extra
Review Comment:
How does Spark handle this?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
