alamb commented on code in PR #14918:
URL: https://github.com/apache/datafusion/pull/14918#discussion_r1975866768
##########
datafusion/core/src/datasource/listing/table.rs:
##########
@@ -1670,6 +1749,56 @@ mod tests {
Ok(())
}
+ /// Check that the files listed by the table match the specified
`output_partitioning`
+ /// when the object store contains `files`, and validate that file
metadata is fetched
+ /// concurrently
+ async fn assert_list_files_for_exact_paths(
+ files: &[&str],
+ target_partitions: usize,
+ output_partitioning: usize,
+ file_ext: Option<&str>,
+ ) -> Result<()> {
+ let ctx = SessionContext::new();
+ let (store, _) = make_test_store_and_state(
+ &files.iter().map(|f| (*f, 10)).collect::<Vec<_>>(),
+ );
+
+ let meta_fetch_concurrency = ctx
+ .state()
+ .config_options()
+ .execution
+ .meta_fetch_concurrency;
+ let expected_concurrency = files.len().min(meta_fetch_concurrency);
+ let head_blocking_store = ensure_head_concurrency(store,
expected_concurrency);
Review Comment:
this is very cool
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
