saadtajwar commented on code in PR #23231:
URL: https://github.com/apache/datafusion/pull/23231#discussion_r3488969136
##########
datafusion/physical-plan/src/repartition/mod.rs:
##########
@@ -826,6 +863,40 @@ impl BatchPartitioner {
// Finished building index-arrays for output partitions
timer.done();
+ let partitioned_batches =
+ Self::partition_grouped_take(&batch, indices,
&self.timer)?;
+
+ Box::new(partitioned_batches.into_iter())
+ }
+ BatchPartitionerState::Range {
+ ordering,
+ split_points,
+ indices,
+ partition_buffer,
+ } => {
+ // Tracking time required for distributing indexes across
output partitions
+ let timer = self.timer.timer();
+
+ let arrays = evaluate_expressions_to_arrays(
+ ordering.iter().map(|e| &e.expr),
+ &batch,
+ )?;
+
+ indices.iter_mut().for_each(|v| v.clear());
+ let sort_options: Vec<SortOptions> =
Review Comment:
I could create this at construction time to avoid re-creating on every
invocation of `partition_iter`?
##########
datafusion/physical-plan/src/repartition/mod.rs:
##########
@@ -826,6 +863,40 @@ impl BatchPartitioner {
// Finished building index-arrays for output partitions
timer.done();
+ let partitioned_batches =
+ Self::partition_grouped_take(&batch, indices,
&self.timer)?;
+
+ Box::new(partitioned_batches.into_iter())
+ }
+ BatchPartitionerState::Range {
+ ordering,
+ split_points,
+ indices,
+ partition_buffer,
+ } => {
+ // Tracking time required for distributing indexes across
output partitions
+ let timer = self.timer.timer();
+
+ let arrays = evaluate_expressions_to_arrays(
+ ordering.iter().map(|e| &e.expr),
+ &batch,
+ )?;
+
+ indices.iter_mut().for_each(|v| v.clear());
+ let sort_options: Vec<SortOptions> =
Review Comment:
I could create this at construction time to avoid re-creating on every
invocation of `partition_iter`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
