SteNicholas commented on code in PR #1449:
URL: https://github.com/apache/paimon-rust/pull/1449#discussion_r4620820965


##########
crates/paimon/src/arrow/format/avro.rs:
##########
@@ -52,66 +67,381 @@ impl FormatFileReader for AvroFormatReader {
         batch_size: Option<usize>,
         row_selection: Option<Vec<RowRange>>,
     ) -> crate::Result<ArrowRecordBatchStream> {
-        // NOTE: Avro OCF requires sequential reading, so we load the entire 
file into memory.
-        // This is fine for typical Paimon data files but may be problematic 
for very large files.
-        let file_bytes = reader.read(0..file_size).await?;
-
-        // Widen the decoded schema to include any predicate columns that are 
not
-        // in the projection, so residual filtering can see them. 
DataFileReader
-        // projects the returned batch to `read_fields` by name, dropping 
extras.
+        // Residual filters need unprojected predicate columns; DataFileReader
+        // projects the filtered batch back to the requested fields.
         let scan_fields = 
crate::arrow::residual::widen_scan_fields(read_fields, predicates);
         let target_schema = build_target_arrow_schema(&scan_fields)?;
-        let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
-        // Own the predicates so the returned 'static stream does not borrow 
the
-        // caller's `&FilePredicates` (FilePredicates is not `Clone`; rebuild 
it).
+        let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
         let predicates = predicates.map(|fp| FilePredicates {
             predicates: fp.predicates.clone(),
             row_filter_factory: None,
             file_fields: fp.file_fields.clone(),
         });
 
-        // Collect Avro records directly as apache_avro::Value, avoiding 
intermediate conversion.
-        let all_records: Vec<Value> = Reader::new(&file_bytes[..])
-            .map_err(|e| Error::UnexpectedError {
-                message: format!("Failed to open Avro file: {e}"),
-                source: Some(Box::new(e)),
-            })?
-            .collect::<std::result::Result<Vec<Value>, _>>()
-            .map_err(|e| Error::UnexpectedError {
-                message: format!("Failed to deserialize Avro record: {e}"),
-                source: Some(Box::new(e)),
-            })?;
+        let context = Arc::new(AvroReadContext {
+            fields: scan_fields,
+            schema: target_schema,
+            predicates,
+            batch_size,
+        });
+        let mut selection = AvroRowSelection::new(row_selection);
+
+        Ok(try_stream! {
+            if selection.is_finished() {
+                return;
+            }
+            let mut input = BufferedAvroInput::new(reader, file_size);
+            let (header, marker) = input.read_header().await?;
+            let header = avro_blocking(move || {
+                // Validate header-only files too; selected blocks use the same
+                // apache-avro schema resolution, logical types and codecs.
+                Reader::new(header.as_slice()).map_err(avro_decode_error)?;
+                Ok(header)
+            }).await?;
+            let mut position = 0_u64;
+            while input.position < file_size && !selection.is_finished() {
+                let mut framing = Vec::with_capacity(20);
+                let count = input.read_length(&mut framing).await?;
+                let size = input.read_length(&mut framing).await?;
+                let end = position.checked_add(count)
+                    .ok_or_else(|| avro_invalid("Avro row position 
overflow"))?;
+                let block_bytes = size.checked_add(16)
+                    .ok_or_else(|| avro_invalid("Avro block size overflow"))?;
+                input.ensure_available(block_bytes)?;
+
+                if count == 0 || !selection.overlaps(position, end) {
+                    // Block counts advance physical positions without 
decoding.
+                    input.skip(size)?;
+                    input.read_marker(&marker).await?;
+                } else {
+                    let mut encoded = header.clone();
+                    encoded.extend_from_slice(&framing);
+                    input.read_into(block_bytes, &mut encoded).await?;
+                    check_avro_marker(&encoded[encoded.len() - 
marker.len()..], &marker)?;
+                    let context = context.clone();
+                    let (mut decoder, mut batch) = avro_blocking(move || {
+                        AvroBatchDecoder {
+                            reader: Reader::new(AvroBlockInput { bytes: 
encoded.into() })
+                                .map_err(avro_decode_error)?,
+                            position,
+                            end,
+                            selection,
+                            context,
+                        }.next_batch()
+                    }).await?;
+                    loop {
+                        if let Some(batch) = batch.take() {
+                            yield batch;
+                        }
+                        if decoder.position == end || 
decoder.selection.is_finished() {
+                            selection = decoder.selection;
+                            break;
+                        }
+                        (decoder, batch) = avro_blocking(move || 
decoder.next_batch()).await?;
+                    }
+                }
+                position = end;
+            }
+        }
+        .boxed())
+    }
+}
 
-        // Apply row selection filtering.
-        let records: Vec<Value> = match row_selection {
-            Some(ref ranges) => {
-                let total_rows = all_records.len();
-                let mask = ranges_to_mask(total_rows, ranges);
-                all_records
+struct AvroReadContext {
+    fields: Vec<DataField>,
+    schema: SchemaRef,
+    predicates: Option<FilePredicates>,
+    batch_size: usize,
+}
+
+/// Release the encoded block after apache-avro copies its payload and marker.
+struct AvroBlockInput {
+    bytes: Bytes,
+}
+
+impl Read for AvroBlockInput {
+    fn read(&mut self, output: &mut [u8]) -> std::io::Result<usize> {
+        let count = output.len().min(self.bytes.len());
+        output[..count].copy_from_slice(&self.bytes[..count]);
+        self.bytes.advance(count);
+        if self.bytes.is_empty() {
+            self.bytes = Bytes::new();
+        }
+        Ok(count)
+    }
+}
+
+struct AvroBatchDecoder {
+    reader: Reader<'static, AvroBlockInput>,
+    position: u64,
+    end: u64,
+    selection: AvroRowSelection,
+    context: Arc<AvroReadContext>,
+}
+
+impl AvroBatchDecoder {
+    fn next_batch(mut self) -> crate::Result<(Self, Option<RecordBatch>)> {
+        let count = (self.end - self.position).min(self.context.batch_size as 
u64);
+        let mut records = Vec::with_capacity(count as usize);
+        // Bound physical decoding too: sparse ranges must not turn a single
+        // non-cancellable blocking task into an entire-block record scan.
+        for _ in 0..count {
+            if self.selection.is_finished() {
+                break;
+            }
+            let record = self
+                .reader
+                .next()
+                .ok_or_else(|| avro_invalid("Truncated Avro block"))?
+                .map_err(avro_decode_error)?;
+            if self.selection.contains(self.position) {
+                records.push(record);
+            }
+            self.position += 1;
+            self.selection.advance_to(self.position);
+        }
+        let batch = if records.is_empty() {
+            None
+        } else {
+            let context = &self.context;
+            let batch = records_to_batch(&records, &context.fields, 
&context.schema)?;
+            Some(match context.predicates.as_ref() {
+                Some(fp) => 
crate::arrow::residual::filter_record_batch_by_predicates(
+                    batch,
+                    fp,
+                    &context.fields,
+                )?,
+                None => batch,
+            })
+        };
+        Ok((self, batch))
+    }
+}
+
+async fn avro_blocking<T: Send + 'static>(
+    task: impl FnOnce() -> crate::Result<T> + Send + 'static,
+) -> crate::Result<T> {
+    tokio::task::spawn_blocking(task)
+        .await
+        .map_err(|error| Error::UnexpectedError {
+            message: "Avro decoding task failed".to_string(),
+            source: Some(Box::new(error)),
+        })?
+}
+
+fn avro_invalid(message: &str) -> Error {
+    Error::DataInvalid {
+        message: message.to_string(),
+        source: None,
+    }
+}
+
+fn avro_decode_error(error: apache_avro::Error) -> Error {
+    Error::UnexpectedError {
+        message: format!("Failed to decode Avro file: {error}"),
+        source: Some(Box::new(error)),
+    }
+}
+
+fn check_avro_marker(actual: &[u8], expected: &[u8; 16]) -> crate::Result<()> {
+    if actual != expected {
+        return Err(avro_invalid("Avro sync marker mismatch"));
+    }
+    Ok(())
+}
+
+/// Sorted inclusive ranges, consumed in original physical row order.
+struct AvroRowSelection {
+    ranges: Option<VecDeque<RowRange>>,
+}
+
+impl AvroRowSelection {
+    fn new(ranges: Option<Vec<RowRange>>) -> Self {
+        Self {
+            ranges: ranges.map(|ranges| {
+                crate::table::merge_row_ranges(ranges)
                     .into_iter()
-                    .enumerate()
-                    .filter(|(i, _)| mask[*i])
-                    .map(|(_, r)| r)
+                    .filter(|range| range.to() >= 0)
+                    .map(|range| RowRange::new(range.from().max(0), 
range.to()))
                     .collect()
+            }),
+        }
+    }
+
+    fn advance_to(&mut self, position: u64) {
+        if let Some(ranges) = &mut self.ranges {
+            while ranges
+                .front()
+                .is_some_and(|range| (range.to() as u64) < position)
+            {
+                ranges.pop_front();
             }
-            None => all_records,
-        };
+        }
+    }
 
-        Ok(try_stream! {
-            for chunk in records.chunks(batch_size) {
-                let batch = records_to_batch(chunk, &scan_fields, 
&target_schema)?;
-                let batch = match predicates.as_ref() {
-                    Some(fp) => 
crate::arrow::residual::filter_record_batch_by_predicates(
-                        batch,
-                        fp,
-                        &scan_fields,
-                    )?,
-                    None => batch,
-                };
-                yield batch;
+    fn is_finished(&self) -> bool {
+        self.ranges.as_ref().is_some_and(VecDeque::is_empty)
+    }
+
+    fn overlaps(&mut self, start: u64, end: u64) -> bool {
+        self.advance_to(start);
+        match &self.ranges {
+            None => true,
+            Some(ranges) => ranges
+                .front()
+                .is_some_and(|range| (range.from() as u64) < end),
+        }
+    }
+
+    fn contains(&mut self, position: u64) -> bool {
+        self.advance_to(position);
+        match &self.ranges {
+            None => true,
+            Some(ranges) => ranges
+                .front()
+                .is_some_and(|range| (range.from() as u64) <= position),
+        }
+    }
+}
+
+/// Small sequential read-ahead buffer. Skipping an OCF payload advances the
+/// offset without fetching it, retaining any already buffered bytes after it.
+struct BufferedAvroInput {
+    reader: Box<dyn FileRead>,
+    file_size: u64,
+    position: u64,
+    buffer: Bytes,
+}
+
+impl BufferedAvroInput {
+    fn new(reader: Box<dyn FileRead>, file_size: u64) -> Self {
+        Self {
+            reader,
+            file_size,
+            position: 0,
+            buffer: Bytes::new(),
+        }
+    }
+
+    fn ensure_available(&self, len: u64) -> crate::Result<()> {
+        if len > self.file_size - self.position {
+            return Err(avro_invalid("Truncated Avro file"));
+        }
+        Ok(())
+    }
+
+    async fn fill(&mut self) -> crate::Result<()> {
+        if self.buffer.is_empty() {
+            self.ensure_available(1)?;
+            let end = self
+                .position
+                .saturating_add(READ_BUFFER_SIZE)
+                .min(self.file_size);
+            self.buffer = self.reader.read(self.position..end).await?;

Review Comment:
   Addressed in b7600be6. Reads start with 64 KiB of read-ahead; known block 
payloads are fetched in requests of up to 8 MiB. After the first batch, 
unrestricted scans grow their read-ahead window to 8 MiB, while explicit row 
ranges retain the smaller window. 
`full_scan_coalesces_reads_for_the_first_block_and_subsequent_blocks` checks 
both the first-block request count and the subsequent full-scan requests, 
alongside the existing prefix/drop test. The PR description includes the 
earlier before/after measurements and identifies the measured revisions.



##########
crates/paimon/src/arrow/format/avro.rs:
##########
@@ -52,66 +67,381 @@ impl FormatFileReader for AvroFormatReader {
         batch_size: Option<usize>,
         row_selection: Option<Vec<RowRange>>,
     ) -> crate::Result<ArrowRecordBatchStream> {
-        // NOTE: Avro OCF requires sequential reading, so we load the entire 
file into memory.
-        // This is fine for typical Paimon data files but may be problematic 
for very large files.
-        let file_bytes = reader.read(0..file_size).await?;
-
-        // Widen the decoded schema to include any predicate columns that are 
not
-        // in the projection, so residual filtering can see them. 
DataFileReader
-        // projects the returned batch to `read_fields` by name, dropping 
extras.
+        // Residual filters need unprojected predicate columns; DataFileReader
+        // projects the filtered batch back to the requested fields.
         let scan_fields = 
crate::arrow::residual::widen_scan_fields(read_fields, predicates);
         let target_schema = build_target_arrow_schema(&scan_fields)?;
-        let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
-        // Own the predicates so the returned 'static stream does not borrow 
the
-        // caller's `&FilePredicates` (FilePredicates is not `Clone`; rebuild 
it).
+        let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
         let predicates = predicates.map(|fp| FilePredicates {
             predicates: fp.predicates.clone(),
             row_filter_factory: None,
             file_fields: fp.file_fields.clone(),
         });
 
-        // Collect Avro records directly as apache_avro::Value, avoiding 
intermediate conversion.
-        let all_records: Vec<Value> = Reader::new(&file_bytes[..])
-            .map_err(|e| Error::UnexpectedError {
-                message: format!("Failed to open Avro file: {e}"),
-                source: Some(Box::new(e)),
-            })?
-            .collect::<std::result::Result<Vec<Value>, _>>()
-            .map_err(|e| Error::UnexpectedError {
-                message: format!("Failed to deserialize Avro record: {e}"),
-                source: Some(Box::new(e)),
-            })?;
+        let context = Arc::new(AvroReadContext {
+            fields: scan_fields,
+            schema: target_schema,
+            predicates,
+            batch_size,
+        });
+        let mut selection = AvroRowSelection::new(row_selection);
+
+        Ok(try_stream! {
+            if selection.is_finished() {
+                return;
+            }
+            let mut input = BufferedAvroInput::new(reader, file_size);
+            let (header, marker) = input.read_header().await?;
+            let header = avro_blocking(move || {
+                // Validate header-only files too; selected blocks use the same
+                // apache-avro schema resolution, logical types and codecs.
+                Reader::new(header.as_slice()).map_err(avro_decode_error)?;
+                Ok(header)
+            }).await?;
+            let mut position = 0_u64;
+            while input.position < file_size && !selection.is_finished() {
+                let mut framing = Vec::with_capacity(20);
+                let count = input.read_length(&mut framing).await?;
+                let size = input.read_length(&mut framing).await?;
+                let end = position.checked_add(count)
+                    .ok_or_else(|| avro_invalid("Avro row position 
overflow"))?;
+                let block_bytes = size.checked_add(16)
+                    .ok_or_else(|| avro_invalid("Avro block size overflow"))?;
+                input.ensure_available(block_bytes)?;
+
+                if count == 0 || !selection.overlaps(position, end) {
+                    // Block counts advance physical positions without 
decoding.
+                    input.skip(size)?;
+                    input.read_marker(&marker).await?;
+                } else {
+                    let mut encoded = header.clone();
+                    encoded.extend_from_slice(&framing);
+                    input.read_into(block_bytes, &mut encoded).await?;
+                    check_avro_marker(&encoded[encoded.len() - 
marker.len()..], &marker)?;
+                    let context = context.clone();
+                    let (mut decoder, mut batch) = avro_blocking(move || {
+                        AvroBatchDecoder {
+                            reader: Reader::new(AvroBlockInput { bytes: 
encoded.into() })
+                                .map_err(avro_decode_error)?,
+                            position,
+                            end,
+                            selection,
+                            context,
+                        }.next_batch()
+                    }).await?;
+                    loop {
+                        if let Some(batch) = batch.take() {
+                            yield batch;
+                        }
+                        if decoder.position == end || 
decoder.selection.is_finished() {
+                            selection = decoder.selection;
+                            break;
+                        }
+                        (decoder, batch) = avro_blocking(move || 
decoder.next_batch()).await?;
+                    }
+                }
+                position = end;
+            }
+        }
+        .boxed())
+    }
+}
 
-        // Apply row selection filtering.
-        let records: Vec<Value> = match row_selection {
-            Some(ref ranges) => {
-                let total_rows = all_records.len();
-                let mask = ranges_to_mask(total_rows, ranges);
-                all_records
+struct AvroReadContext {
+    fields: Vec<DataField>,
+    schema: SchemaRef,
+    predicates: Option<FilePredicates>,
+    batch_size: usize,
+}
+
+/// Release the encoded block after apache-avro copies its payload and marker.
+struct AvroBlockInput {
+    bytes: Bytes,
+}
+
+impl Read for AvroBlockInput {
+    fn read(&mut self, output: &mut [u8]) -> std::io::Result<usize> {
+        let count = output.len().min(self.bytes.len());
+        output[..count].copy_from_slice(&self.bytes[..count]);
+        self.bytes.advance(count);
+        if self.bytes.is_empty() {
+            self.bytes = Bytes::new();
+        }
+        Ok(count)
+    }
+}
+
+struct AvroBatchDecoder {
+    reader: Reader<'static, AvroBlockInput>,
+    position: u64,
+    end: u64,
+    selection: AvroRowSelection,
+    context: Arc<AvroReadContext>,
+}
+
+impl AvroBatchDecoder {
+    fn next_batch(mut self) -> crate::Result<(Self, Option<RecordBatch>)> {
+        let count = (self.end - self.position).min(self.context.batch_size as 
u64);
+        let mut records = Vec::with_capacity(count as usize);
+        // Bound physical decoding too: sparse ranges must not turn a single
+        // non-cancellable blocking task into an entire-block record scan.
+        for _ in 0..count {
+            if self.selection.is_finished() {
+                break;
+            }
+            let record = self
+                .reader
+                .next()
+                .ok_or_else(|| avro_invalid("Truncated Avro block"))?
+                .map_err(avro_decode_error)?;
+            if self.selection.contains(self.position) {
+                records.push(record);
+            }
+            self.position += 1;
+            self.selection.advance_to(self.position);
+        }
+        let batch = if records.is_empty() {
+            None
+        } else {
+            let context = &self.context;
+            let batch = records_to_batch(&records, &context.fields, 
&context.schema)?;
+            Some(match context.predicates.as_ref() {
+                Some(fp) => 
crate::arrow::residual::filter_record_batch_by_predicates(
+                    batch,
+                    fp,
+                    &context.fields,
+                )?,
+                None => batch,
+            })
+        };
+        Ok((self, batch))
+    }
+}
+
+async fn avro_blocking<T: Send + 'static>(
+    task: impl FnOnce() -> crate::Result<T> + Send + 'static,
+) -> crate::Result<T> {
+    tokio::task::spawn_blocking(task)

Review Comment:
   Addressed in b7600be6. `run_decode_task` checks `Handle::try_current()` and 
decodes inline when there is no Tokio runtime, yielding between physical 
batches. The fallback also converts decoder panics into stream errors. Tests 
read a Java Avro fixture through memory-backed FileIO on 
`futures::executor::block_on`, cover corrupt payloads on both executors, and 
verify cancellation during a sparse selection with fully buffered input.



##########
crates/paimon/src/arrow/format/avro.rs:
##########
@@ -52,66 +67,381 @@ impl FormatFileReader for AvroFormatReader {
         batch_size: Option<usize>,
         row_selection: Option<Vec<RowRange>>,
     ) -> crate::Result<ArrowRecordBatchStream> {
-        // NOTE: Avro OCF requires sequential reading, so we load the entire 
file into memory.
-        // This is fine for typical Paimon data files but may be problematic 
for very large files.
-        let file_bytes = reader.read(0..file_size).await?;
-
-        // Widen the decoded schema to include any predicate columns that are 
not
-        // in the projection, so residual filtering can see them. 
DataFileReader
-        // projects the returned batch to `read_fields` by name, dropping 
extras.
+        // Residual filters need unprojected predicate columns; DataFileReader
+        // projects the filtered batch back to the requested fields.
         let scan_fields = 
crate::arrow::residual::widen_scan_fields(read_fields, predicates);
         let target_schema = build_target_arrow_schema(&scan_fields)?;
-        let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
-        // Own the predicates so the returned 'static stream does not borrow 
the
-        // caller's `&FilePredicates` (FilePredicates is not `Clone`; rebuild 
it).
+        let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
         let predicates = predicates.map(|fp| FilePredicates {
             predicates: fp.predicates.clone(),
             row_filter_factory: None,
             file_fields: fp.file_fields.clone(),
         });
 
-        // Collect Avro records directly as apache_avro::Value, avoiding 
intermediate conversion.
-        let all_records: Vec<Value> = Reader::new(&file_bytes[..])
-            .map_err(|e| Error::UnexpectedError {
-                message: format!("Failed to open Avro file: {e}"),
-                source: Some(Box::new(e)),
-            })?
-            .collect::<std::result::Result<Vec<Value>, _>>()
-            .map_err(|e| Error::UnexpectedError {
-                message: format!("Failed to deserialize Avro record: {e}"),
-                source: Some(Box::new(e)),
-            })?;
+        let context = Arc::new(AvroReadContext {
+            fields: scan_fields,
+            schema: target_schema,
+            predicates,
+            batch_size,
+        });
+        let mut selection = AvroRowSelection::new(row_selection);
+
+        Ok(try_stream! {
+            if selection.is_finished() {
+                return;
+            }
+            let mut input = BufferedAvroInput::new(reader, file_size);
+            let (header, marker) = input.read_header().await?;
+            let header = avro_blocking(move || {
+                // Validate header-only files too; selected blocks use the same
+                // apache-avro schema resolution, logical types and codecs.
+                Reader::new(header.as_slice()).map_err(avro_decode_error)?;
+                Ok(header)
+            }).await?;
+            let mut position = 0_u64;
+            while input.position < file_size && !selection.is_finished() {
+                let mut framing = Vec::with_capacity(20);
+                let count = input.read_length(&mut framing).await?;
+                let size = input.read_length(&mut framing).await?;
+                let end = position.checked_add(count)
+                    .ok_or_else(|| avro_invalid("Avro row position 
overflow"))?;
+                let block_bytes = size.checked_add(16)
+                    .ok_or_else(|| avro_invalid("Avro block size overflow"))?;
+                input.ensure_available(block_bytes)?;
+
+                if count == 0 || !selection.overlaps(position, end) {
+                    // Block counts advance physical positions without 
decoding.
+                    input.skip(size)?;
+                    input.read_marker(&marker).await?;
+                } else {
+                    let mut encoded = header.clone();
+                    encoded.extend_from_slice(&framing);
+                    input.read_into(block_bytes, &mut encoded).await?;
+                    check_avro_marker(&encoded[encoded.len() - 
marker.len()..], &marker)?;
+                    let context = context.clone();
+                    let (mut decoder, mut batch) = avro_blocking(move || {
+                        AvroBatchDecoder {
+                            reader: Reader::new(AvroBlockInput { bytes: 
encoded.into() })

Review Comment:
   Addressed in b7600be6. `AvroBatchDecoder` creates one `apache_avro::Reader` 
after reading the header and keeps it for the file. `AvroBlockInput` supplies 
successive selected blocks to that reader, so the writer schema and resolved 
names are reused across blocks. The codec and nested-value tests exercise 
multiple blocks, including named record references. The PR description retains 
the earlier comparison results with their exact revision and fixture details.



##########
crates/paimon/src/arrow/format/avro.rs:
##########
@@ -52,66 +67,381 @@ impl FormatFileReader for AvroFormatReader {
         batch_size: Option<usize>,
         row_selection: Option<Vec<RowRange>>,
     ) -> crate::Result<ArrowRecordBatchStream> {
-        // NOTE: Avro OCF requires sequential reading, so we load the entire 
file into memory.
-        // This is fine for typical Paimon data files but may be problematic 
for very large files.
-        let file_bytes = reader.read(0..file_size).await?;
-
-        // Widen the decoded schema to include any predicate columns that are 
not
-        // in the projection, so residual filtering can see them. 
DataFileReader
-        // projects the returned batch to `read_fields` by name, dropping 
extras.
+        // Residual filters need unprojected predicate columns; DataFileReader
+        // projects the filtered batch back to the requested fields.
         let scan_fields = 
crate::arrow::residual::widen_scan_fields(read_fields, predicates);
         let target_schema = build_target_arrow_schema(&scan_fields)?;
-        let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
-        // Own the predicates so the returned 'static stream does not borrow 
the
-        // caller's `&FilePredicates` (FilePredicates is not `Clone`; rebuild 
it).
+        let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
         let predicates = predicates.map(|fp| FilePredicates {
             predicates: fp.predicates.clone(),
             row_filter_factory: None,
             file_fields: fp.file_fields.clone(),
         });
 
-        // Collect Avro records directly as apache_avro::Value, avoiding 
intermediate conversion.
-        let all_records: Vec<Value> = Reader::new(&file_bytes[..])
-            .map_err(|e| Error::UnexpectedError {
-                message: format!("Failed to open Avro file: {e}"),
-                source: Some(Box::new(e)),
-            })?
-            .collect::<std::result::Result<Vec<Value>, _>>()
-            .map_err(|e| Error::UnexpectedError {
-                message: format!("Failed to deserialize Avro record: {e}"),
-                source: Some(Box::new(e)),
-            })?;
+        let context = Arc::new(AvroReadContext {
+            fields: scan_fields,
+            schema: target_schema,
+            predicates,
+            batch_size,
+        });
+        let mut selection = AvroRowSelection::new(row_selection);
+
+        Ok(try_stream! {
+            if selection.is_finished() {
+                return;
+            }
+            let mut input = BufferedAvroInput::new(reader, file_size);
+            let (header, marker) = input.read_header().await?;
+            let header = avro_blocking(move || {
+                // Validate header-only files too; selected blocks use the same
+                // apache-avro schema resolution, logical types and codecs.
+                Reader::new(header.as_slice()).map_err(avro_decode_error)?;
+                Ok(header)
+            }).await?;
+            let mut position = 0_u64;
+            while input.position < file_size && !selection.is_finished() {
+                let mut framing = Vec::with_capacity(20);
+                let count = input.read_length(&mut framing).await?;
+                let size = input.read_length(&mut framing).await?;
+                let end = position.checked_add(count)
+                    .ok_or_else(|| avro_invalid("Avro row position 
overflow"))?;
+                let block_bytes = size.checked_add(16)
+                    .ok_or_else(|| avro_invalid("Avro block size overflow"))?;
+                input.ensure_available(block_bytes)?;
+
+                if count == 0 || !selection.overlaps(position, end) {
+                    // Block counts advance physical positions without 
decoding.
+                    input.skip(size)?;
+                    input.read_marker(&marker).await?;
+                } else {
+                    let mut encoded = header.clone();
+                    encoded.extend_from_slice(&framing);
+                    input.read_into(block_bytes, &mut encoded).await?;
+                    check_avro_marker(&encoded[encoded.len() - 
marker.len()..], &marker)?;
+                    let context = context.clone();
+                    let (mut decoder, mut batch) = avro_blocking(move || {
+                        AvroBatchDecoder {
+                            reader: Reader::new(AvroBlockInput { bytes: 
encoded.into() })
+                                .map_err(avro_decode_error)?,
+                            position,
+                            end,
+                            selection,
+                            context,
+                        }.next_batch()
+                    }).await?;
+                    loop {
+                        if let Some(batch) = batch.take() {
+                            yield batch;
+                        }
+                        if decoder.position == end || 
decoder.selection.is_finished() {
+                            selection = decoder.selection;
+                            break;
+                        }
+                        (decoder, batch) = avro_blocking(move || 
decoder.next_batch()).await?;
+                    }
+                }
+                position = end;
+            }
+        }
+        .boxed())
+    }
+}
 
-        // Apply row selection filtering.
-        let records: Vec<Value> = match row_selection {
-            Some(ref ranges) => {
-                let total_rows = all_records.len();
-                let mask = ranges_to_mask(total_rows, ranges);
-                all_records
+struct AvroReadContext {
+    fields: Vec<DataField>,
+    schema: SchemaRef,
+    predicates: Option<FilePredicates>,
+    batch_size: usize,
+}
+
+/// Release the encoded block after apache-avro copies its payload and marker.
+struct AvroBlockInput {
+    bytes: Bytes,
+}
+
+impl Read for AvroBlockInput {
+    fn read(&mut self, output: &mut [u8]) -> std::io::Result<usize> {
+        let count = output.len().min(self.bytes.len());
+        output[..count].copy_from_slice(&self.bytes[..count]);
+        self.bytes.advance(count);
+        if self.bytes.is_empty() {
+            self.bytes = Bytes::new();
+        }
+        Ok(count)
+    }
+}
+
+struct AvroBatchDecoder {
+    reader: Reader<'static, AvroBlockInput>,
+    position: u64,
+    end: u64,
+    selection: AvroRowSelection,
+    context: Arc<AvroReadContext>,
+}
+
+impl AvroBatchDecoder {
+    fn next_batch(mut self) -> crate::Result<(Self, Option<RecordBatch>)> {
+        let count = (self.end - self.position).min(self.context.batch_size as 
u64);
+        let mut records = Vec::with_capacity(count as usize);
+        // Bound physical decoding too: sparse ranges must not turn a single
+        // non-cancellable blocking task into an entire-block record scan.
+        for _ in 0..count {
+            if self.selection.is_finished() {
+                break;
+            }
+            let record = self
+                .reader
+                .next()
+                .ok_or_else(|| avro_invalid("Truncated Avro block"))?
+                .map_err(avro_decode_error)?;
+            if self.selection.contains(self.position) {
+                records.push(record);
+            }
+            self.position += 1;
+            self.selection.advance_to(self.position);
+        }
+        let batch = if records.is_empty() {
+            None
+        } else {
+            let context = &self.context;
+            let batch = records_to_batch(&records, &context.fields, 
&context.schema)?;
+            Some(match context.predicates.as_ref() {
+                Some(fp) => 
crate::arrow::residual::filter_record_batch_by_predicates(
+                    batch,
+                    fp,
+                    &context.fields,
+                )?,
+                None => batch,
+            })
+        };
+        Ok((self, batch))
+    }
+}
+
+async fn avro_blocking<T: Send + 'static>(
+    task: impl FnOnce() -> crate::Result<T> + Send + 'static,
+) -> crate::Result<T> {
+    tokio::task::spawn_blocking(task)
+        .await

Review Comment:
   Addressed in b7600be6. The blocking task is wrapped in `AbortOnDropHandle`, 
and its queued closure holds only a weak reference to the pending input. 
Dropping the waiting future therefore aborts queued work and releases its 
captured data without waiting for a blocking worker. 
`dropping_decode_future_releases_and_aborts_queued_work` occupies the sole 
blocking thread, queues a decode, drops it, verifies the captured state is 
released, then frees the worker and confirms the decode never ran. A task that 
has already started may finish its current block decompression and physical 
batch.



##########
crates/paimon/src/arrow/format/avro.rs:
##########
@@ -52,66 +67,381 @@ impl FormatFileReader for AvroFormatReader {
         batch_size: Option<usize>,
         row_selection: Option<Vec<RowRange>>,
     ) -> crate::Result<ArrowRecordBatchStream> {
-        // NOTE: Avro OCF requires sequential reading, so we load the entire 
file into memory.
-        // This is fine for typical Paimon data files but may be problematic 
for very large files.
-        let file_bytes = reader.read(0..file_size).await?;
-
-        // Widen the decoded schema to include any predicate columns that are 
not
-        // in the projection, so residual filtering can see them. 
DataFileReader
-        // projects the returned batch to `read_fields` by name, dropping 
extras.
+        // Residual filters need unprojected predicate columns; DataFileReader
+        // projects the filtered batch back to the requested fields.
         let scan_fields = 
crate::arrow::residual::widen_scan_fields(read_fields, predicates);
         let target_schema = build_target_arrow_schema(&scan_fields)?;
-        let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
-        // Own the predicates so the returned 'static stream does not borrow 
the
-        // caller's `&FilePredicates` (FilePredicates is not `Clone`; rebuild 
it).
+        let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
         let predicates = predicates.map(|fp| FilePredicates {
             predicates: fp.predicates.clone(),
             row_filter_factory: None,
             file_fields: fp.file_fields.clone(),
         });
 
-        // Collect Avro records directly as apache_avro::Value, avoiding 
intermediate conversion.
-        let all_records: Vec<Value> = Reader::new(&file_bytes[..])
-            .map_err(|e| Error::UnexpectedError {
-                message: format!("Failed to open Avro file: {e}"),
-                source: Some(Box::new(e)),
-            })?
-            .collect::<std::result::Result<Vec<Value>, _>>()
-            .map_err(|e| Error::UnexpectedError {
-                message: format!("Failed to deserialize Avro record: {e}"),
-                source: Some(Box::new(e)),
-            })?;
+        let context = Arc::new(AvroReadContext {
+            fields: scan_fields,
+            schema: target_schema,
+            predicates,
+            batch_size,
+        });
+        let mut selection = AvroRowSelection::new(row_selection);
+
+        Ok(try_stream! {
+            if selection.is_finished() {
+                return;
+            }
+            let mut input = BufferedAvroInput::new(reader, file_size);
+            let (header, marker) = input.read_header().await?;
+            let header = avro_blocking(move || {
+                // Validate header-only files too; selected blocks use the same
+                // apache-avro schema resolution, logical types and codecs.
+                Reader::new(header.as_slice()).map_err(avro_decode_error)?;
+                Ok(header)
+            }).await?;
+            let mut position = 0_u64;
+            while input.position < file_size && !selection.is_finished() {
+                let mut framing = Vec::with_capacity(20);
+                let count = input.read_length(&mut framing).await?;
+                let size = input.read_length(&mut framing).await?;
+                let end = position.checked_add(count)
+                    .ok_or_else(|| avro_invalid("Avro row position 
overflow"))?;
+                let block_bytes = size.checked_add(16)
+                    .ok_or_else(|| avro_invalid("Avro block size overflow"))?;
+                input.ensure_available(block_bytes)?;
+
+                if count == 0 || !selection.overlaps(position, end) {
+                    // Block counts advance physical positions without 
decoding.
+                    input.skip(size)?;
+                    input.read_marker(&marker).await?;
+                } else {
+                    let mut encoded = header.clone();
+                    encoded.extend_from_slice(&framing);
+                    input.read_into(block_bytes, &mut encoded).await?;
+                    check_avro_marker(&encoded[encoded.len() - 
marker.len()..], &marker)?;

Review Comment:
   Addressed in b7600be6. The scan now uses an owning `try_unfold` step, so 
read/decode state is dropped before a terminal error is yielded. The shared 
`terminal_error` assertion retains the errored stream and verifies both that 
the FileRead was dropped and that its storage window has no remaining stream 
owner, before polling again. This covers marker mismatches, malformed/truncated 
input, codec errors and injected I/O failures.



##########
crates/paimon/src/arrow/format/avro/streaming_tests/benchmark.rs:
##########
@@ -0,0 +1,185 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::*;
+use std::path::PathBuf;
+use std::sync::atomic::AtomicU64;
+use std::time::Instant;
+use tokio::io::{AsyncReadExt, AsyncSeekExt};
+
+const BLOCK_SIZE: usize = 1024 * 1024;
+const BATCH_SIZE: usize = 1024;
+const ROWS: usize = 2_000_000;
+const PAYLOAD_SIZE: usize = 128;
+
+struct BenchmarkRead {
+    file: tokio::sync::Mutex<tokio::fs::File>,
+    bytes_read: Arc<AtomicU64>,
+}
+
+#[async_trait]
+impl FileRead for BenchmarkRead {
+    async fn read(&self, range: Range<u64>) -> crate::Result<Bytes> {
+        let mut file = self.file.lock().await;
+        let mut buffer = vec![0; (range.end - range.start) as usize];
+        file.seek(std::io::SeekFrom::Start(range.start))
+            .await
+            .map_err(io_error)?;
+        file.read_exact(&mut buffer).await.map_err(io_error)?;
+        self.bytes_read
+            .fetch_add(buffer.len() as u64, Ordering::Relaxed);
+        Ok(buffer.into())
+    }
+}
+
+fn io_error(error: std::io::Error) -> Error {
+    Error::UnexpectedError {
+        message: "Avro benchmark I/O failed".into(),
+        source: Some(Box::new(error)),
+    }
+}
+
+async fn eager_stream(
+    reader: Box<dyn FileRead>,
+    size: u64,
+) -> crate::Result<ArrowRecordBatchStream> {
+    let bytes = reader.read(0..size).await?;
+    let records = Reader::new(bytes.as_ref())
+        .map_err(avro_decode_error)?
+        .collect::<std::result::Result<Vec<_>, _>>()
+        .map_err(avro_decode_error)?;
+    let fields = fields();
+    let schema = build_target_arrow_schema(&fields)?;
+    Ok(try_stream! {
+        for records in records.chunks(BATCH_SIZE) {
+            yield records_to_batch(records, &fields, &schema)?;
+        }
+    }
+    .boxed())
+}
+
+fn peak_rss_kib() -> Option<u64> {
+    std::fs::read_to_string("/proc/self/status")
+        .ok()?
+        .lines()
+        .find_map(|line| {
+            line.strip_prefix("VmHWM:")?
+                .split_whitespace()
+                .next()?
+                .parse()
+                .ok()
+        })
+}
+
+#[tokio::test]
+#[ignore = "manual Avro full-scan/LIMIT comparison; see 
benchmarks/avro/README.md"]
+async fn compare_avro_reads() {

Review Comment:
   Addressed in b7600be6. Removed `streaming_tests/benchmark.rs` and its module 
declaration. The README now documents the streaming behavior and links to the 
PR description, which contains the existing full-scan and LIMIT measurements, 
fixture recipe, cache policy and measurement method. Those figures remain 
explicitly attributed to the earlier measured revisions; the latest head's 83 
Avro tests, full unit suite, formatting and Clippy results are reported 
separately.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to