This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new ec3d20bff6 Adds instrumentation to delimited LIST operations in CLI
(#18134)
ec3d20bff6 is described below
commit ec3d20bff60511a9a7a3822a271af8fbbac7ddca
Author: Blake Orth <[email protected]>
AuthorDate: Fri Oct 17 11:46:10 2025 -0600
Adds instrumentation to delimited LIST operations in CLI (#18134)
## Which issue does this PR close?
This does not fully close, but is an incremental building block
component for:
- https://github.com/apache/datafusion/issues/17207
The full context of how this code is likely to progress can be seen in
the POC for this effort:
- https://github.com/apache/datafusion/pull/17266
## Rationale for this change
Continued progress filling out methods that are instrumented by the
instrumented object store
## What changes are included in this PR?
- Adds instrumentation around delimited list operations into the
instrumented object store
- Adds test cases for the new code
## Are these changes tested?
Yes, unit tests have been added.
Example output:
```sql
DataFusion CLI v50.2.0
> CREATE EXTERNAL TABLE overture_partitioned
STORED AS PARQUET LOCATION
's3://overturemaps-us-west-2/release/2025-09-24.0/theme=addresses/';
0 row(s) fetched.
Elapsed 2.307 seconds.
> \object_store_profiling trace
ObjectStore Profile mode set to Trace
> select count(*) from overture_partitioned;
+-----------+
| count(*) |
+-----------+
| 446544475 |
+-----------+
1 row(s) fetched.
Elapsed 1.932 seconds.
Object Store Profiling
Instrumented Object Store: instrument_mode: Trace, inner:
AmazonS3(overturemaps-us-west-2)
2025-10-17T17:05:27.922724180+00:00 operation=List duration=0.132154s
path=release/2025-09-24.0/theme=addresses
2025-10-17T17:05:28.054894440+00:00 operation=List duration=0.049048s
path=release/2025-09-24.0/theme=addresses/type=address
2025-10-17T17:05:28.104233937+00:00 operation=Get duration=0.053522s size=8
range: bytes=1070778162-1070778169
path=release/2025-09-24.0/theme=addresses/type=address/part-00000-52872134-68de-44a6-822d-15fa29a0f606-c000.zstd.parquet
2025-10-17T17:05:28.106862343+00:00 operation=Get duration=0.108103s size=8
range: bytes=1017940335-1017940342
path=release/2025-09-24.0/theme=addresses/type=address/part-00003-52872134-68de-44a6-822d-15fa29a0f606-c000.zstd.parquet
...
2025-10-17T17:05:28.589084204+00:00 operation=Get duration=0.084737s
size=836971 range: bytes=1112791717-1113628687
path=release/2025-09-24.0/theme=addresses/type=address/part-00009-52872134-68de-44a6-822d-15fa29a0f606-c000.zstd.parquet
Summaries:
List
count: 2
duration min: 0.049048s
duration max: 0.132154s
duration avg: 0.090601s
Get
count: 33
duration min: 0.045500s
duration max: 0.162114s
duration avg: 0.089775s
size min: 8 B
size max: 917946 B
size avg: 336000 B
size sum: 11088026 B
>
```
Note that a `LIST` report showing a duration must be a
`list_with_delimiter()` call because a standard `list` call does not
currently report a duration.
## Are there any user-facing changes?
No-ish
cc @alamb
---
datafusion-cli/src/object_storage/instrumented.rs | 49 +++++++++++++++++++++++
1 file changed, 49 insertions(+)
diff --git a/datafusion-cli/src/object_storage/instrumented.rs
b/datafusion-cli/src/object_storage/instrumented.rs
index 8acece315f..94445ee64e 100644
--- a/datafusion-cli/src/object_storage/instrumented.rs
+++ b/datafusion-cli/src/object_storage/instrumented.rs
@@ -163,6 +163,28 @@ impl InstrumentedObjectStore {
ret
}
+
+ async fn instrumented_list_with_delimiter(
+ &self,
+ prefix: Option<&Path>,
+ ) -> Result<ListResult> {
+ let timestamp = Utc::now();
+ let start = Instant::now();
+ let ret = self.inner.list_with_delimiter(prefix).await?;
+ let elapsed = start.elapsed();
+
+ self.requests.lock().push(RequestDetails {
+ op: Operation::List,
+ path: prefix.cloned().unwrap_or_else(|| Path::from("")),
+ timestamp,
+ duration: Some(elapsed),
+ size: None,
+ range: None,
+ extra_display: None,
+ });
+
+ Ok(ret)
+ }
}
impl fmt::Display for InstrumentedObjectStore {
@@ -217,6 +239,10 @@ impl ObjectStore for InstrumentedObjectStore {
}
async fn list_with_delimiter(&self, prefix: Option<&Path>) ->
Result<ListResult> {
+ if self.enabled() {
+ return self.instrumented_list_with_delimiter(prefix).await;
+ }
+
self.inner.list_with_delimiter(prefix).await
}
@@ -569,6 +595,29 @@ mod tests {
assert!(request.extra_display.is_none());
}
+ #[tokio::test]
+ async fn instrumented_store_list_with_delimiter() {
+ let (instrumented, path) = setup_test_store().await;
+
+ // By default no requests should be instrumented/stored
+ assert!(instrumented.requests.lock().is_empty());
+ let _ = instrumented.list_with_delimiter(Some(&path)).await.unwrap();
+ assert!(instrumented.requests.lock().is_empty());
+
+ instrumented.set_instrument_mode(InstrumentedObjectStoreMode::Trace);
+ assert!(instrumented.requests.lock().is_empty());
+ let _ = instrumented.list_with_delimiter(Some(&path)).await.unwrap();
+ assert_eq!(instrumented.requests.lock().len(), 1);
+
+ let request = instrumented.take_requests().pop().unwrap();
+ assert_eq!(request.op, Operation::List);
+ assert_eq!(request.path, path);
+ assert!(request.duration.is_some());
+ assert!(request.size.is_none());
+ assert!(request.range.is_none());
+ assert!(request.extra_display.is_none());
+ }
+
#[test]
fn request_details() {
let rd = RequestDetails {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
