lhoestq commented on issue #706:
URL:
https://github.com/apache/arrow-rs-object-store/issues/706#issuecomment-4730187250
Now that OpenDAL 0.57 is out you can use `object_store_opendal` to get an
`object_store::ObjectStore` to access HF Buckets :)
with code in the lines of
```rust
let opendal_op = Operator::from_uri("hf://buckets/xxx/yyy")?;
let store = std::sync::Arc::new(OpendalStore::new(opendal_op));
// Example: stream-based files listing
let mut stream = store.list(Some(&Path::ROOT));
```
Full example below
<details>
```rust
use futures::StreamExt;
use object_store::path::Path;
use object_store::ObjectStore;
use object_store_opendal::OpendalStore;
use opendal::Operator;
#[tokio::main]
async fn main() -> object_store::Result<()> {
// Create an OpenDAL operator from the hf:// URL
let opendal_op = Operator::from_uri("hf://buckets/lhoestq/datasets")
.map_err(|e| object_store::Error::Generic {
store: "opendal",
source: Box::new(e),
})?;
// Wrap it in an object_store::ObjectStore via OpendalStore
let store: std::sync::Arc<dyn ObjectStore> =
std::sync::Arc::new(OpendalStore::new(opendal_op));
println!("Using object_store: {:?}", store);
println!();
// List files in the bucket
println!("=== Listing files ===");
let prefix = Path::ROOT;
let mut list_stream = store.list(Some(&prefix));
let mut count = 0usize;
while let Some(meta) = list_stream.next().await {
let meta = meta?;
println!(
" {:<60} size: {:>10}",
meta.location,
meta.size
);
count += 1;
}
println!("Total entries: {}", count);
println!();
Ok(())
}
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
