Dandandan commented on code in PR #3593:
URL: https://github.com/apache/arrow-datafusion/pull/3593#discussion_r979252095
##########
datafusion/core/src/physical_plan/sorts/sort.rs:
##########
@@ -1062,6 +1077,65 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn test_sort_fetch_memory_calculation() -> Result<()> {
+ // This test mirrors down the size from the example above.
+ let avg_batch_size = 5336;
+ let partitions = 4;
+
+ // A tuple of (fetch, expect_spillage)
+ let test_options = vec![
+ // Since we don't have a limit (and the memory is less than the
total size of
+ // all the batches we are processing, we expect it to spill.
+ (None, true),
+ // When we have a limit however, the buffered size of batches
should fit in memory
+ // since it is much lover than the total size of the input batch.
+ (Some(1), false),
+ ];
+
+ for (fetch, expect_spillage) in test_options {
+ let config = RuntimeConfig::new()
+ .with_memory_limit(avg_batch_size * (partitions - 1), 1.0);
Review Comment:
👍
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
