This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 65f9726f75 feat: Expose cache hits in list_files_cache function
(#23439)
65f9726f75 is described below
commit 65f9726f753bea6a13ca7f7356eede0127aa4b87
Author: Michael Kleen <[email protected]>
AuthorDate: Mon Jul 13 13:38:50 2026 +0200
feat: Expose cache hits in list_files_cache function (#23439)
## Which issue does this PR close?
- Closes None.
## Rationale for this change
Follow up to https://github.com/apache/datafusion/pull/22613 and
https://github.com/apache/datafusion/pull/23253. Cache hits are now
supported for all memory-limiting caches. Therefore it makes sense to
expose them in the `list_files_cache` function the same way the
`metadata_cache` and `statistics_cache` functions do it already.
## What changes are included in this PR?
- Add cache hits to the `list_files_cache` function
- Tests
- Adapt documentation for the `list_files_cache` function
## Are these changes tested?
Yes.
## Are there any user-facing changes?
Yes, the `list_files_cache` function supports now cache hits but no
breaking changes.
---
datafusion-cli/src/functions.rs | 4 ++++
datafusion-cli/src/main.rs | 15 ++++++++-------
docs/source/user-guide/cli/functions.md | 1 +
3 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/datafusion-cli/src/functions.rs b/datafusion-cli/src/functions.rs
index 7d87e7ed8a..164af2559d 100644
--- a/datafusion-cli/src/functions.rs
+++ b/datafusion-cli/src/functions.rs
@@ -813,6 +813,7 @@ impl TableFunctionImpl for ListFilesCacheFunc {
DataType::List(Arc::new(metadata_field.clone())),
true,
),
+ Field::new("hits", DataType::UInt64, false),
]));
let mut table_arr = vec![];
@@ -826,6 +827,7 @@ impl TableFunctionImpl for ListFilesCacheFunc {
let mut etag_arr = vec![];
let mut version_arr = vec![];
let mut offsets: Vec<i32> = vec![0];
+ let mut hits_arr = vec![];
if let Some(list_files_cache) =
self.cache_manager.get_list_files_cache() {
let now = Instant::now();
@@ -851,6 +853,7 @@ impl TableFunctionImpl for ListFilesCacheFunc {
}
current_offset += entry.value.files.len() as i32;
offsets.push(current_offset);
+ hits_arr.push(entry.hits as u64);
}
}
@@ -882,6 +885,7 @@ impl TableFunctionImpl for ListFilesCacheFunc {
Arc::new(struct_arr),
None,
)),
+ Arc::new(UInt64Array::from(hits_arr)),
],
)?;
diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs
index 78d8342020..20a2537d7c 100644
--- a/datafusion-cli/src/main.rs
+++ b/datafusion-cli/src/main.rs
@@ -810,7 +810,7 @@ mod tests {
.collect()
.await?;
- let sql = "SELECT metadata_size_bytes, expires_in, metadata_list FROM
list_files_cache()";
+ let sql = "SELECT metadata_size_bytes, expires_in, metadata_list, hits
FROM list_files_cache()";
let df = ctx
.sql(sql)
.await?
@@ -838,16 +838,17 @@ mod tests {
"filename",
"file_size_bytes",
"etag",
+ "hits",
])?
.sort(vec![col("filename").sort(true, false)])?;
let rbs = df.collect().await?;
assert_snapshot!(batches_to_string(&rbs),@r"
- +---------------------+-----------+-----------------+------+
- | metadata_size_bytes | filename | file_size_bytes | etag |
- +---------------------+-----------+-----------------+------+
- | 212 | 0.parquet | 3642 | 0 |
- | 212 | 1.parquet | 3642 | 1 |
- +---------------------+-----------+-----------------+------+
+ +---------------------+-----------+-----------------+------+------+
+ | metadata_size_bytes | filename | file_size_bytes | etag | hits |
+ +---------------------+-----------+-----------------+------+------+
+ | 212 | 0.parquet | 3642 | 0 | 2 |
+ | 212 | 1.parquet | 3642 | 1 | 2 |
+ +---------------------+-----------+-----------------+------+------+
");
Ok(())
diff --git a/docs/source/user-guide/cli/functions.md
b/docs/source/user-guide/cli/functions.md
index 409661ac82..baf054ef5a 100644
--- a/docs/source/user-guide/cli/functions.md
+++ b/docs/source/user-guide/cli/functions.md
@@ -208,6 +208,7 @@ The columns of the returned table are:
| path | Utf8 | File path relative to the object store
/ filesystem root |
| metadata_size_bytes | UInt64 | Size of the cached metadata in memory
(not its thrift encoded form) |
| expires_in | Duration(ms) | Last modified time of the file
|
+| hits | UInt64 | Number of times the cached metadata has
been accessed |
| metadata_list | List(Struct) | List of metadatas, one for each file
under the path. |
A metadata struct in the metadata_list contains the following fields:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
