kosiew commented on code in PR #27043:
URL: https://github.com/apache/datafusion/pull/27043#discussion_r4640000784
##########
datafusion/physical-plan/src/sorts/sort.rs:
##########
@@ -552,6 +552,63 @@ impl ExternalSorter {
"in_mem_batches must not be empty when attempting to sort and
spill"
);
+ let has_expanding_keys = self.expr.iter().try_fold(false, |found,
expr| {
+ let data_type = expr.expr.data_type(&self.schema)?;
+ Ok::<_, DataFusionError>(
+ found
+ || !(data_type.is_primitive()
+ || matches!(
+ data_type,
+ DataType::Boolean
+ | DataType::Null
+ | DataType::FixedSizeBinary(_)
+ )),
+ )
+ })?;
+ if has_expanding_keys
Review Comment:
The gate also admits short `Utf8` keys, where per-batch spill files can
exhaust the temp-disk quota through repeated metadata and extra merge passes. A
4,096-row sort succeeded under a 300 KiB quota without this branch but now
fails. Please narrow admission or avoid this disk overhead, and add a
regression test.
##########
datafusion/physical-plan/src/sorts/sort.rs:
##########
@@ -552,6 +552,63 @@ impl ExternalSorter {
"in_mem_batches must not be empty when attempting to sort and
spill"
);
+ let has_expanding_keys = self.expr.iter().try_fold(false, |found,
expr| {
Review Comment:
Consider checking key count, batch count, and reservation thresholds before
querying expression types. This avoids unnecessary `data_type` calls when
independent spilling is already ruled out.
##########
datafusion/physical-plan/src/sorts/sort.rs:
##########
@@ -2914,6 +3000,122 @@ mod tests {
Ok(())
}
+ #[rstest::rstest]
+ #[case::no_headroom(0, 128)]
+ #[case::reserved_headroom_chunked(256 * 1024, 47)]
+ fn test_spill_wide_dictionary_keys(
+ #[case] merge_headroom: usize,
+ #[case] batch_size: usize,
+ #[values(false, true)] multithreaded: bool,
+ ) -> Result<()> {
+ const KEY_COLUMNS: usize = 32;
+ const VALUE_COUNT: usize = 32;
+ const ROWS_PER_BATCH: usize = 128;
+ const BATCH_COUNT: usize = 32;
+ // Row-format merge keys expand these compact dictionaries enough to
+ // exhaust the pool if all buffered runs are merged before spilling.
+ let values = (0..VALUE_COUNT)
+ .map(|i| format!("{i:04}{}", "x".repeat(64)))
+ .collect::<Vec<_>>();
+ let keys = (0..KEY_COLUMNS)
+ .map(|column| {
+ Arc::new(DictionaryArray::<UInt32Type>::from_iter(
+ (0..ROWS_PER_BATCH)
+ .map(|row| Some(values[(row + column) %
VALUE_COUNT].as_str())),
+ )) as ArrayRef
+ })
+ .collect::<Vec<_>>();
+ let batches = (0..BATCH_COUNT)
+ .map(|batch| {
+ let start = batch * ROWS_PER_BATCH;
+ let mut columns = keys.clone();
+ columns.push(Arc::new(Int64Array::from_iter_values(
+ (start..start + ROWS_PER_BATCH).rev().map(|id| id as i64),
+ )));
+ RecordBatch::try_from_iter(
+ columns
+ .into_iter()
+ .enumerate()
+ .map(|(i, array)| (format!("c{i}"), array)),
+ )
+ })
+ .collect::<std::result::Result<Vec<_>, _>>()?;
+ let schema = batches[0].schema();
+ let input = TestMemoryExec::try_new_exec(&[batches],
Arc::clone(&schema), None)?;
+ let expressions = (0..=KEY_COLUMNS)
+ .map(|i| {
+
PhysicalSortExpr::new_default(Arc::new(Column::new(&format!("c{i}"), i)))
+ })
+ .collect::<Vec<_>>();
+ let sort =
Arc::new(SortExec::new(LexOrdering::new(expressions).unwrap(), input));
+ let pool: Arc<dyn MemoryPool> = Arc::new(FairSpillPool::new(4 * 1024 *
1024));
+ let runtime = RuntimeEnvBuilder::new()
+ .with_memory_pool(Arc::clone(&pool))
+ .with_max_spill_merge_fan_in(2)
+ .build_arc()?;
+ let task_ctx = Arc::new(
+ TaskContext::default()
+ .with_session_config(
+ SessionConfig::new()
+ .with_batch_size(batch_size)
+ .with_sort_spill_reservation_bytes(merge_headroom),
+ )
+ .with_runtime(runtime),
+ );
+ let mut builder = if multithreaded {
+ let mut builder = tokio::runtime::Builder::new_multi_thread();
+ builder.worker_threads(2);
+ builder
+ } else {
+ tokio::runtime::Builder::new_current_thread()
+ };
+ let executor = builder.enable_all().build()?;
+ let output = executor.block_on(collect(
+ Arc::clone(&sort) as Arc<dyn ExecutionPlan>,
+ task_ctx,
+ ))?;
+ assert_eq!(output[0].schema(), schema);
+ let decoded = output
+ .iter()
+ .map(|batch| {
+ RecordBatch::try_from_iter(
+ batch
+ .columns()
+ .iter()
+ .enumerate()
+ .map(|(i, array)| {
+ let array = if i < KEY_COLUMNS {
+ arrow::compute::cast(array, &DataType::Utf8)
+ } else {
+ Ok(Arc::clone(array))
+ };
+ array.map(|array| (format!("c{i}"), array))
+ })
+ .collect::<std::result::Result<Vec<_>, _>>()?,
+ )
+ })
+ .collect::<std::result::Result<Vec<_>, _>>()?;
+ let actual = concat_batches(&decoded[0].schema(), &decoded)?;
+ let key_index =
Review Comment:
Consider varying the second dictionary key independently among equal first
keys across runs. The current test cannot detect a merge that ignores later
dictionary keys. Verify ordering by `(first_key, second_key, id)`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]