This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23439-04b19a3732c7706d9199d5b075f7aac7f78b1c6d in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 65f9726f753bea6a13ca7f7356eede0127aa4b87 Author: Michael Kleen <[email protected]> AuthorDate: Mon Jul 13 13:38:50 2026 +0200 feat: Expose cache hits in list_files_cache function (#23439) ## Which issue does this PR close? - Closes None. ## Rationale for this change Follow up to https://github.com/apache/datafusion/pull/22613 and https://github.com/apache/datafusion/pull/23253. Cache hits are now supported for all memory-limiting caches. Therefore it makes sense to expose them in the `list_files_cache` function the same way the `metadata_cache` and `statistics_cache` functions do it already. ## What changes are included in this PR? - Add cache hits to the `list_files_cache` function - Tests - Adapt documentation for the `list_files_cache` function ## Are these changes tested? Yes. ## Are there any user-facing changes? Yes, the `list_files_cache` function supports now cache hits but no breaking changes. --- datafusion-cli/src/functions.rs | 4 ++++ datafusion-cli/src/main.rs | 15 ++++++++------- docs/source/user-guide/cli/functions.md | 1 + 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/datafusion-cli/src/functions.rs b/datafusion-cli/src/functions.rs index 7d87e7ed8a..164af2559d 100644 --- a/datafusion-cli/src/functions.rs +++ b/datafusion-cli/src/functions.rs @@ -813,6 +813,7 @@ impl TableFunctionImpl for ListFilesCacheFunc { DataType::List(Arc::new(metadata_field.clone())), true, ), + Field::new("hits", DataType::UInt64, false), ])); let mut table_arr = vec![]; @@ -826,6 +827,7 @@ impl TableFunctionImpl for ListFilesCacheFunc { let mut etag_arr = vec![]; let mut version_arr = vec![]; let mut offsets: Vec<i32> = vec![0]; + let mut hits_arr = vec![]; if let Some(list_files_cache) = self.cache_manager.get_list_files_cache() { let now = Instant::now(); @@ -851,6 +853,7 @@ impl TableFunctionImpl for ListFilesCacheFunc { } current_offset += entry.value.files.len() as i32; offsets.push(current_offset); + hits_arr.push(entry.hits as u64); } } @@ -882,6 +885,7 @@ impl TableFunctionImpl for ListFilesCacheFunc { Arc::new(struct_arr), None, )), + Arc::new(UInt64Array::from(hits_arr)), ], )?; diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index 78d8342020..20a2537d7c 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -810,7 +810,7 @@ mod tests { .collect() .await?; - let sql = "SELECT metadata_size_bytes, expires_in, metadata_list FROM list_files_cache()"; + let sql = "SELECT metadata_size_bytes, expires_in, metadata_list, hits FROM list_files_cache()"; let df = ctx .sql(sql) .await? @@ -838,16 +838,17 @@ mod tests { "filename", "file_size_bytes", "etag", + "hits", ])? .sort(vec![col("filename").sort(true, false)])?; let rbs = df.collect().await?; assert_snapshot!(batches_to_string(&rbs),@r" - +---------------------+-----------+-----------------+------+ - | metadata_size_bytes | filename | file_size_bytes | etag | - +---------------------+-----------+-----------------+------+ - | 212 | 0.parquet | 3642 | 0 | - | 212 | 1.parquet | 3642 | 1 | - +---------------------+-----------+-----------------+------+ + +---------------------+-----------+-----------------+------+------+ + | metadata_size_bytes | filename | file_size_bytes | etag | hits | + +---------------------+-----------+-----------------+------+------+ + | 212 | 0.parquet | 3642 | 0 | 2 | + | 212 | 1.parquet | 3642 | 1 | 2 | + +---------------------+-----------+-----------------+------+------+ "); Ok(()) diff --git a/docs/source/user-guide/cli/functions.md b/docs/source/user-guide/cli/functions.md index 409661ac82..baf054ef5a 100644 --- a/docs/source/user-guide/cli/functions.md +++ b/docs/source/user-guide/cli/functions.md @@ -208,6 +208,7 @@ The columns of the returned table are: | path | Utf8 | File path relative to the object store / filesystem root | | metadata_size_bytes | UInt64 | Size of the cached metadata in memory (not its thrift encoded form) | | expires_in | Duration(ms) | Last modified time of the file | +| hits | UInt64 | Number of times the cached metadata has been accessed | | metadata_list | List(Struct) | List of metadatas, one for each file under the path. | A metadata struct in the metadata_list contains the following fields: --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
