rluvaton commented on code in PR #15700:
URL: https://github.com/apache/datafusion/pull/15700#discussion_r2045582560
##########
datafusion/physical-plan/src/sorts/sort.rs:
##########
@@ -431,12 +422,16 @@ impl ExternalSorter {
let batches_to_spill = std::mem::take(globally_sorted_batches);
self.reservation.free();
- let in_progress_file =
self.in_progress_spill_file.as_mut().ok_or_else(|| {
- internal_datafusion_err!("In-progress spill file should be
initialized")
- })?;
+ let (in_progress_file, max_record_batch_size) =
+ self.in_progress_spill_file.as_mut().ok_or_else(|| {
+ internal_datafusion_err!("In-progress spill file should be
initialized")
+ })?;
for batch in batches_to_spill {
in_progress_file.append_batch(&batch)?;
+
+ *max_record_batch_size =
+ (*max_record_batch_size).max(batch.get_actually_used_size());
Review Comment:
So after looking at the code I came to the conclusion that this is still the
closest there is to accurately estimating memory
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
