kosiew commented on code in PR #27043:
URL: https://github.com/apache/datafusion/pull/27043#discussion_r4640000784


##########
datafusion/physical-plan/src/sorts/sort.rs:
##########
@@ -552,6 +552,63 @@ impl ExternalSorter {
             "in_mem_batches must not be empty when attempting to sort and 
spill"
         );
 
+        let has_expanding_keys = self.expr.iter().try_fold(false, |found, 
expr| {
+            let data_type = expr.expr.data_type(&self.schema)?;
+            Ok::<_, DataFusionError>(
+                found
+                    || !(data_type.is_primitive()
+                        || matches!(
+                            data_type,
+                            DataType::Boolean
+                                | DataType::Null
+                                | DataType::FixedSizeBinary(_)
+                        )),
+            )
+        })?;
+        if has_expanding_keys

Review Comment:
   The gate also admits short `Utf8` keys, where per-batch spill files can 
exhaust the temp-disk quota through repeated metadata and extra merge passes. A 
4,096-row sort succeeded under a 300 KiB quota without this branch but now 
fails. Please narrow admission or avoid this disk overhead, and add a 
regression test.



##########
datafusion/physical-plan/src/sorts/sort.rs:
##########
@@ -552,6 +552,63 @@ impl ExternalSorter {
             "in_mem_batches must not be empty when attempting to sort and 
spill"
         );
 
+        let has_expanding_keys = self.expr.iter().try_fold(false, |found, 
expr| {

Review Comment:
   Consider checking key count, batch count, and reservation thresholds before 
querying expression types. This avoids unnecessary `data_type` calls when 
independent spilling is already ruled out.



##########
datafusion/physical-plan/src/sorts/sort.rs:
##########
@@ -2914,6 +3000,122 @@ mod tests {
         Ok(())
     }
 
+    #[rstest::rstest]
+    #[case::no_headroom(0, 128)]
+    #[case::reserved_headroom_chunked(256 * 1024, 47)]
+    fn test_spill_wide_dictionary_keys(
+        #[case] merge_headroom: usize,
+        #[case] batch_size: usize,
+        #[values(false, true)] multithreaded: bool,
+    ) -> Result<()> {
+        const KEY_COLUMNS: usize = 32;
+        const VALUE_COUNT: usize = 32;
+        const ROWS_PER_BATCH: usize = 128;
+        const BATCH_COUNT: usize = 32;
+        // Row-format merge keys expand these compact dictionaries enough to
+        // exhaust the pool if all buffered runs are merged before spilling.
+        let values = (0..VALUE_COUNT)
+            .map(|i| format!("{i:04}{}", "x".repeat(64)))
+            .collect::<Vec<_>>();
+        let keys = (0..KEY_COLUMNS)
+            .map(|column| {
+                Arc::new(DictionaryArray::<UInt32Type>::from_iter(
+                    (0..ROWS_PER_BATCH)
+                        .map(|row| Some(values[(row + column) % 
VALUE_COUNT].as_str())),
+                )) as ArrayRef
+            })
+            .collect::<Vec<_>>();
+        let batches = (0..BATCH_COUNT)
+            .map(|batch| {
+                let start = batch * ROWS_PER_BATCH;
+                let mut columns = keys.clone();
+                columns.push(Arc::new(Int64Array::from_iter_values(
+                    (start..start + ROWS_PER_BATCH).rev().map(|id| id as i64),
+                )));
+                RecordBatch::try_from_iter(
+                    columns
+                        .into_iter()
+                        .enumerate()
+                        .map(|(i, array)| (format!("c{i}"), array)),
+                )
+            })
+            .collect::<std::result::Result<Vec<_>, _>>()?;
+        let schema = batches[0].schema();
+        let input = TestMemoryExec::try_new_exec(&[batches], 
Arc::clone(&schema), None)?;
+        let expressions = (0..=KEY_COLUMNS)
+            .map(|i| {
+                
PhysicalSortExpr::new_default(Arc::new(Column::new(&format!("c{i}"), i)))
+            })
+            .collect::<Vec<_>>();
+        let sort = 
Arc::new(SortExec::new(LexOrdering::new(expressions).unwrap(), input));
+        let pool: Arc<dyn MemoryPool> = Arc::new(FairSpillPool::new(4 * 1024 * 
1024));
+        let runtime = RuntimeEnvBuilder::new()
+            .with_memory_pool(Arc::clone(&pool))
+            .with_max_spill_merge_fan_in(2)
+            .build_arc()?;
+        let task_ctx = Arc::new(
+            TaskContext::default()
+                .with_session_config(
+                    SessionConfig::new()
+                        .with_batch_size(batch_size)
+                        .with_sort_spill_reservation_bytes(merge_headroom),
+                )
+                .with_runtime(runtime),
+        );
+        let mut builder = if multithreaded {
+            let mut builder = tokio::runtime::Builder::new_multi_thread();
+            builder.worker_threads(2);
+            builder
+        } else {
+            tokio::runtime::Builder::new_current_thread()
+        };
+        let executor = builder.enable_all().build()?;
+        let output = executor.block_on(collect(
+            Arc::clone(&sort) as Arc<dyn ExecutionPlan>,
+            task_ctx,
+        ))?;
+        assert_eq!(output[0].schema(), schema);
+        let decoded = output
+            .iter()
+            .map(|batch| {
+                RecordBatch::try_from_iter(
+                    batch
+                        .columns()
+                        .iter()
+                        .enumerate()
+                        .map(|(i, array)| {
+                            let array = if i < KEY_COLUMNS {
+                                arrow::compute::cast(array, &DataType::Utf8)
+                            } else {
+                                Ok(Arc::clone(array))
+                            };
+                            array.map(|array| (format!("c{i}"), array))
+                        })
+                        .collect::<std::result::Result<Vec<_>, _>>()?,
+                )
+            })
+            .collect::<std::result::Result<Vec<_>, _>>()?;
+        let actual = concat_batches(&decoded[0].schema(), &decoded)?;
+        let key_index =

Review Comment:
   Consider varying the second dictionary key independently among equal first 
keys across runs. The current test cannot detect a merge that ignores later 
dictionary keys. Verify ordering by `(first_key, second_key, id)`.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to