adriangb commented on code in PR #27602:
URL: https://github.com/apache/datafusion/pull/27602#discussion_r4639089941
##########
datafusion/execution/src/disk_manager.rs:
##########
@@ -619,6 +627,56 @@ fn escape_for_temp_file_name_prefix(text: &str) -> String {
mod tests {
use super::*;
+ /// Counts the open file descriptors of this process.
+ #[cfg(unix)]
+ fn open_fd_count() -> usize {
+ std::fs::read_dir("/dev/fd").unwrap().count()
+ }
+
+ /// Live spill files must not hold file descriptors. Only open writers and
+ /// readers may hold one, so the descriptor count stays bounded however
+ /// many spill files an operator keeps.
+ #[cfg(unix)]
+ #[test]
+ fn live_spill_files_hold_no_file_descriptors() -> Result<()> {
+ use std::io::Write;
+ // More than the macOS default soft limit of 256 descriptors.
+ const FILES: usize = 400;
+ // Other tests in this process can open files concurrently.
+ const SLACK: usize = 64;
+
+ let dm = Arc::new(DiskManagerBuilder::default().build()?);
+ // Create the temp directory before the first count.
+ drop(dm.create_tmp_file("warmup")?);
+ let before = open_fd_count();
+
+ let mut files = Vec::with_capacity(FILES);
+ for _ in 0..FILES {
+ let file = dm.create_tmp_file("fd bound")?;
+ let mut writer = file.open_writer()?;
+ writer.write_all(b"spilled")?;
+ writer.finish()?;
+ drop(writer);
+ files.push(file);
+ }
+
+ let held = open_fd_count().saturating_sub(before);
Review Comment:
what if other tests are opening files?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]