alamb commented on code in PR #24061:
URL: https://github.com/apache/datafusion/pull/24061#discussion_r3728974606
##########
datafusion/physical-plan/src/aggregates/hash_stream.rs:
##########
@@ -804,119 +1125,305 @@ impl FinalHashAggregateStream {
fn handle_reading_input(
&mut self,
cx: &mut Context<'_>,
- mut original_state: FinalHashAggregateState,
+ original_state: FinalHashAggregateState,
) -> FinalHashAggregateStateTransition {
- debug_assert!(matches!(
- &original_state,
- FinalHashAggregateState::ReadingInput { .. }
- ));
- debug_assert!(original_state.hash_table().is_building());
+ let FinalHashAggregateState::ReadingInput {
+ mut hash_table,
+ spill_context,
+ } = original_state
+ else {
+ return Self::break_with_internal_err(
+ "Final hash aggregate stream expected ReadingInput state",
+ );
+ };
match self.input.poll_next_unpin(cx) {
- Poll::Pending => ControlFlow::Break((Poll::Pending,
original_state)),
+ Poll::Pending => ControlFlow::Break((
+ Poll::Pending,
+ FinalHashAggregateState::ReadingInput {
+ hash_table,
+ spill_context,
+ },
+ )),
Poll::Ready(Some(Ok(batch))) => {
let elapsed_compute =
self.baseline_metrics.elapsed_compute().clone();
let timer = elapsed_compute.timer();
- let result =
original_state.hash_table_mut().aggregate_batch(&batch);
+ let result = hash_table.aggregate_batch(&batch);
timer.done();
if let Err(e) = result {
- return ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- original_state,
- ));
+ return Self::break_with_err(e);
}
- if self.hit_soft_group_limit(original_state.hash_table()) {
+ if self.hit_soft_group_limit(&hash_table) {
let timer = elapsed_compute.timer();
- let result =
self.start_output(original_state.hash_table_mut());
+ let result = self.start_output(&mut hash_table);
timer.done();
- if let Err(e) = result {
- return ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- original_state,
- ));
- }
-
- return
ControlFlow::Continue(original_state.into_producing_output());
+ return match result {
+ Ok(()) => ControlFlow::Continue(
+ FinalHashAggregateState::ProducingOutput {
hash_table },
+ ),
+ Err(e) => Self::break_with_err(e),
+ };
}
- if let Err(e) = self
- .reservation
- .try_resize(original_state.hash_table().memory_size())
- {
- return ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- original_state,
- ));
+ // Check memory reservation, and potentially spill.
+ let timer = elapsed_compute.timer();
Review Comment:
As in I was thinking that you could avopid `timer.done()` -- as the
destructor also stops the timer I think
##########
datafusion/physical-plan/src/aggregates/hash_stream.rs:
##########
@@ -592,45 +826,42 @@ impl PartialHashAggregateStream {
fn handle_skipping_aggregation(
&mut self,
cx: &mut Context<'_>,
- mut original_state: PartialHashAggregateState,
+ original_state: PartialHashAggregateState,
) -> PartialHashAggregateStateTransition {
- debug_assert!(matches!(
- &original_state,
- PartialHashAggregateState::SkippingAggregation { .. }
- ));
+ let PartialHashAggregateState::SkippingAggregation { mut hash_table } =
Review Comment:
Me neither - this way is fine too
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
