alamb commented on code in PR #24061:
URL: https://github.com/apache/datafusion/pull/24061#discussion_r3723225313
##########
datafusion/physical-plan/src/aggregates/hash_stream.rs:
##########
@@ -804,119 +1125,305 @@ impl FinalHashAggregateStream {
fn handle_reading_input(
&mut self,
cx: &mut Context<'_>,
- mut original_state: FinalHashAggregateState,
+ original_state: FinalHashAggregateState,
) -> FinalHashAggregateStateTransition {
- debug_assert!(matches!(
- &original_state,
- FinalHashAggregateState::ReadingInput { .. }
- ));
- debug_assert!(original_state.hash_table().is_building());
+ let FinalHashAggregateState::ReadingInput {
+ mut hash_table,
+ spill_context,
+ } = original_state
+ else {
+ return Self::break_with_internal_err(
+ "Final hash aggregate stream expected ReadingInput state",
+ );
+ };
match self.input.poll_next_unpin(cx) {
- Poll::Pending => ControlFlow::Break((Poll::Pending,
original_state)),
+ Poll::Pending => ControlFlow::Break((
+ Poll::Pending,
+ FinalHashAggregateState::ReadingInput {
+ hash_table,
+ spill_context,
+ },
+ )),
Poll::Ready(Some(Ok(batch))) => {
let elapsed_compute =
self.baseline_metrics.elapsed_compute().clone();
let timer = elapsed_compute.timer();
- let result =
original_state.hash_table_mut().aggregate_batch(&batch);
+ let result = hash_table.aggregate_batch(&batch);
timer.done();
if let Err(e) = result {
- return ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- original_state,
- ));
+ return Self::break_with_err(e);
}
- if self.hit_soft_group_limit(original_state.hash_table()) {
+ if self.hit_soft_group_limit(&hash_table) {
Review Comment:
Claude code flagged this as a potential bug -- if the limit is set, this
code path appears to simply drop the spill_context and any spilled content thus
far
##########
datafusion/physical-plan/src/aggregates/ordered_final_stream.rs:
##########
@@ -230,14 +230,18 @@ impl OrderedFinalSpillContext {
} = self;
let spill_schema = Arc::clone(spill_manager.schema());
+ // The merge and replay table are two components of the same aggregate
+ // operator. Keep them under one consumer registration so a fair memory
+ // pool does not divide this operator's quota between its own phases.
Review Comment:
is this a behavior change? Or does it mirror what the old operator does?
(It seems reasonable to me, but I wanted to check)
##########
datafusion/physical-plan/src/aggregates/hash_stream.rs:
##########
@@ -777,10 +1058,31 @@ impl FinalHashAggregateStream {
baseline_metrics,
reservation,
group_values_soft_limit: agg.limit_options().map(|config|
config.limit()),
- state: Some(FinalHashAggregateState::ReadingInput { hash_table }),
+ state: Some(FinalHashAggregateState::ReadingInput {
+ hash_table,
+ spill_context,
+ }),
})
}
+ fn close_input(&mut self) {
+ let input_schema = self.input.schema();
+ self.input = Box::pin(EmptyRecordBatchStream::new(input_schema));
+ }
+
+ fn break_with_err(error: DataFusionError) ->
FinalHashAggregateStateTransition {
Review Comment:
this structure is quite elegant and easy to follow 👍
##########
datafusion/physical-plan/src/aggregates/hash_stream.rs:
##########
@@ -592,45 +826,42 @@ impl PartialHashAggregateStream {
fn handle_skipping_aggregation(
&mut self,
cx: &mut Context<'_>,
- mut original_state: PartialHashAggregateState,
+ original_state: PartialHashAggregateState,
) -> PartialHashAggregateStateTransition {
- debug_assert!(matches!(
- &original_state,
- PartialHashAggregateState::SkippingAggregation { .. }
- ));
+ let PartialHashAggregateState::SkippingAggregation { mut hash_table } =
Review Comment:
you could potentially avoid these internal errors by simply passing in
`hash_table` rather than an original_state
The call site already matched on the type of `PartialHashAggregateState`
##########
datafusion/physical-plan/src/aggregates/hash_stream.rs:
##########
@@ -804,119 +1125,305 @@ impl FinalHashAggregateStream {
fn handle_reading_input(
&mut self,
cx: &mut Context<'_>,
- mut original_state: FinalHashAggregateState,
+ original_state: FinalHashAggregateState,
) -> FinalHashAggregateStateTransition {
- debug_assert!(matches!(
- &original_state,
- FinalHashAggregateState::ReadingInput { .. }
- ));
- debug_assert!(original_state.hash_table().is_building());
+ let FinalHashAggregateState::ReadingInput {
+ mut hash_table,
+ spill_context,
+ } = original_state
+ else {
+ return Self::break_with_internal_err(
+ "Final hash aggregate stream expected ReadingInput state",
+ );
+ };
match self.input.poll_next_unpin(cx) {
- Poll::Pending => ControlFlow::Break((Poll::Pending,
original_state)),
+ Poll::Pending => ControlFlow::Break((
+ Poll::Pending,
+ FinalHashAggregateState::ReadingInput {
+ hash_table,
+ spill_context,
+ },
+ )),
Poll::Ready(Some(Ok(batch))) => {
let elapsed_compute =
self.baseline_metrics.elapsed_compute().clone();
let timer = elapsed_compute.timer();
- let result =
original_state.hash_table_mut().aggregate_batch(&batch);
+ let result = hash_table.aggregate_batch(&batch);
timer.done();
if let Err(e) = result {
- return ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- original_state,
- ));
+ return Self::break_with_err(e);
}
- if self.hit_soft_group_limit(original_state.hash_table()) {
+ if self.hit_soft_group_limit(&hash_table) {
let timer = elapsed_compute.timer();
- let result =
self.start_output(original_state.hash_table_mut());
+ let result = self.start_output(&mut hash_table);
timer.done();
- if let Err(e) = result {
- return ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- original_state,
- ));
- }
-
- return
ControlFlow::Continue(original_state.into_producing_output());
+ return match result {
+ Ok(()) => ControlFlow::Continue(
+ FinalHashAggregateState::ProducingOutput {
hash_table },
+ ),
+ Err(e) => Self::break_with_err(e),
+ };
}
- if let Err(e) = self
- .reservation
- .try_resize(original_state.hash_table().memory_size())
- {
- return ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- original_state,
- ));
+ // Check memory reservation, and potentially spill.
+ let timer = elapsed_compute.timer();
Review Comment:
a nit here is that you could potentially use the same `timer` as above
rather than making a new one here
##########
datafusion/physical-plan/src/aggregates/hash_stream.rs:
##########
@@ -804,119 +1125,305 @@ impl FinalHashAggregateStream {
fn handle_reading_input(
&mut self,
cx: &mut Context<'_>,
- mut original_state: FinalHashAggregateState,
+ original_state: FinalHashAggregateState,
) -> FinalHashAggregateStateTransition {
- debug_assert!(matches!(
- &original_state,
- FinalHashAggregateState::ReadingInput { .. }
- ));
- debug_assert!(original_state.hash_table().is_building());
+ let FinalHashAggregateState::ReadingInput {
+ mut hash_table,
+ spill_context,
+ } = original_state
+ else {
+ return Self::break_with_internal_err(
+ "Final hash aggregate stream expected ReadingInput state",
+ );
+ };
match self.input.poll_next_unpin(cx) {
- Poll::Pending => ControlFlow::Break((Poll::Pending,
original_state)),
+ Poll::Pending => ControlFlow::Break((
+ Poll::Pending,
+ FinalHashAggregateState::ReadingInput {
+ hash_table,
+ spill_context,
+ },
+ )),
Poll::Ready(Some(Ok(batch))) => {
let elapsed_compute =
self.baseline_metrics.elapsed_compute().clone();
let timer = elapsed_compute.timer();
- let result =
original_state.hash_table_mut().aggregate_batch(&batch);
+ let result = hash_table.aggregate_batch(&batch);
timer.done();
if let Err(e) = result {
- return ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- original_state,
- ));
+ return Self::break_with_err(e);
}
- if self.hit_soft_group_limit(original_state.hash_table()) {
+ if self.hit_soft_group_limit(&hash_table) {
Review Comment:
Maybe update to
```rust
let spilled = spill_context
.as_ref()
.is_some_and(|ctx| ctx.has_spills());
if self.hit_soft_group_limit(&hash_table) && !spilled {
// existing early-output path
}
```
🤔
But somehow you have to switch to emitting 🤔
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
