pepijnve commented on code in PR #23522:
URL: https://github.com/apache/datafusion/pull/23522#discussion_r3595069616
##########
datafusion/physical-plan/src/repartition/mod.rs:
##########
@@ -168,6 +168,32 @@ struct OutputChannel {
shared_coalescer: Option<SharedCoalescer>,
}
+/// The set of spill-pool writers for a single output partition, before they
are handed to the
+/// per-input tasks. The variant encodes the repartition mode so the wrong
writer topology cannot
+/// be constructed for a given mode.
+enum PartitionSpillWriters {
+ /// `preserve_order`: one single-producer FIFO writer per input partition.
Each is `take`n
+ /// exactly once (moved into the matching input task), so the pool always
has one writer.
+ PerInput(Vec<Option<SpillPoolWriter>>),
+ /// Non-preserve-order: one shared writer, cloned into every input task.
+ Shared(SharedSpillPoolWriter),
+}
+
+impl PartitionSpillWriters {
+ /// Hand out the writer for input partition `input`.
+ ///
+ /// In `PerInput` mode this moves the dedicated writer out (it must only
be requested once per
+ /// input); in `Shared` mode it clones the shared writer.
+ fn take_for_input(&mut self, input: usize) -> SpillPoolWriter {
+ match self {
+ PartitionSpillWriters::PerInput(writers) => writers[input]
+ .take()
+ .expect("spill writer for input partition requested more than
once"),
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
