alamb commented on code in PR #11080: URL: https://github.com/apache/arrow-rs/pull/11080#discussion_r4019203929
########## arrow-avro/examples/opendal.rs: ########## @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Read and write Avro with Apache OpenDAL using an in-memory service. +//! +//! Run with `cargo run -p arrow-avro --example opendal --features async`. +//! Replace the Memory service and enable the corresponding OpenDAL service feature +//! to use remote storage. The adapter uses OpenDAL's native buffer APIs for range reads. +//! The synchronous Avro writer buffers this small file in memory before uploading it. + +use arrow_array::{ArrayRef, Int64Array, RecordBatch}; +use arrow_avro::errors::AvroError; +use arrow_avro::reader::{AsyncAvroFileReader, AsyncFileReader, SpawnedReader}; +use arrow_avro::writer::AvroWriter; +use bytes::Bytes; +use futures::future::BoxFuture; +use futures::{FutureExt, TryStreamExt}; +use opendal::{Operator, Reader, services::Memory}; +use std::ops::Range; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<(), Box<dyn std::error::Error>> { + let operator = Operator::new(Memory::default())?; + let path = "example.avro"; + let col = Arc::new(Int64Array::from(vec![Some(1), None, Some(3)])) as ArrayRef; + let batch = RecordBatch::try_from_iter([("col", col)])?; + + let mut writer = AvroWriter::new(Vec::new(), batch.schema().as_ref().clone())?; + writer.write(&batch)?; + // Flush the final Avro block before uploading the complete file. + writer.finish()?; + operator.write(path, writer.into_inner()).await?; + + let file_size = operator.stat(path).await?.content_length(); + let reader = OpenDalReader(operator.reader(path).await?); + let stream = AsyncAvroFileReader::builder(reader.clone(), file_size, 1024) Review Comment: 👍 looks good to me ########## arrow-avro/Cargo.toml: ########## @@ -95,6 +95,7 @@ futures = "0.3.31" async-stream = "0.3.6" apache-avro = "0.22.0" object_store = { workspace = true, features = ["fs"] } +opendal = { version = "0.59.1", default-features = false, features = ["services-memory"] } Review Comment: Dev dependency 👍 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
