SteNicholas commented on code in PR #1449:
URL: https://github.com/apache/paimon-rust/pull/1449#discussion_r4620820965
##########
crates/paimon/src/arrow/format/avro.rs:
##########
@@ -52,66 +67,381 @@ impl FormatFileReader for AvroFormatReader {
batch_size: Option<usize>,
row_selection: Option<Vec<RowRange>>,
) -> crate::Result<ArrowRecordBatchStream> {
- // NOTE: Avro OCF requires sequential reading, so we load the entire
file into memory.
- // This is fine for typical Paimon data files but may be problematic
for very large files.
- let file_bytes = reader.read(0..file_size).await?;
-
- // Widen the decoded schema to include any predicate columns that are
not
- // in the projection, so residual filtering can see them.
DataFileReader
- // projects the returned batch to `read_fields` by name, dropping
extras.
+ // Residual filters need unprojected predicate columns; DataFileReader
+ // projects the filtered batch back to the requested fields.
let scan_fields =
crate::arrow::residual::widen_scan_fields(read_fields, predicates);
let target_schema = build_target_arrow_schema(&scan_fields)?;
- let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
- // Own the predicates so the returned 'static stream does not borrow
the
- // caller's `&FilePredicates` (FilePredicates is not `Clone`; rebuild
it).
+ let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
let predicates = predicates.map(|fp| FilePredicates {
predicates: fp.predicates.clone(),
row_filter_factory: None,
file_fields: fp.file_fields.clone(),
});
- // Collect Avro records directly as apache_avro::Value, avoiding
intermediate conversion.
- let all_records: Vec<Value> = Reader::new(&file_bytes[..])
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to open Avro file: {e}"),
- source: Some(Box::new(e)),
- })?
- .collect::<std::result::Result<Vec<Value>, _>>()
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to deserialize Avro record: {e}"),
- source: Some(Box::new(e)),
- })?;
+ let context = Arc::new(AvroReadContext {
+ fields: scan_fields,
+ schema: target_schema,
+ predicates,
+ batch_size,
+ });
+ let mut selection = AvroRowSelection::new(row_selection);
+
+ Ok(try_stream! {
+ if selection.is_finished() {
+ return;
+ }
+ let mut input = BufferedAvroInput::new(reader, file_size);
+ let (header, marker) = input.read_header().await?;
+ let header = avro_blocking(move || {
+ // Validate header-only files too; selected blocks use the same
+ // apache-avro schema resolution, logical types and codecs.
+ Reader::new(header.as_slice()).map_err(avro_decode_error)?;
+ Ok(header)
+ }).await?;
+ let mut position = 0_u64;
+ while input.position < file_size && !selection.is_finished() {
+ let mut framing = Vec::with_capacity(20);
+ let count = input.read_length(&mut framing).await?;
+ let size = input.read_length(&mut framing).await?;
+ let end = position.checked_add(count)
+ .ok_or_else(|| avro_invalid("Avro row position
overflow"))?;
+ let block_bytes = size.checked_add(16)
+ .ok_or_else(|| avro_invalid("Avro block size overflow"))?;
+ input.ensure_available(block_bytes)?;
+
+ if count == 0 || !selection.overlaps(position, end) {
+ // Block counts advance physical positions without
decoding.
+ input.skip(size)?;
+ input.read_marker(&marker).await?;
+ } else {
+ let mut encoded = header.clone();
+ encoded.extend_from_slice(&framing);
+ input.read_into(block_bytes, &mut encoded).await?;
+ check_avro_marker(&encoded[encoded.len() -
marker.len()..], &marker)?;
+ let context = context.clone();
+ let (mut decoder, mut batch) = avro_blocking(move || {
+ AvroBatchDecoder {
+ reader: Reader::new(AvroBlockInput { bytes:
encoded.into() })
+ .map_err(avro_decode_error)?,
+ position,
+ end,
+ selection,
+ context,
+ }.next_batch()
+ }).await?;
+ loop {
+ if let Some(batch) = batch.take() {
+ yield batch;
+ }
+ if decoder.position == end ||
decoder.selection.is_finished() {
+ selection = decoder.selection;
+ break;
+ }
+ (decoder, batch) = avro_blocking(move ||
decoder.next_batch()).await?;
+ }
+ }
+ position = end;
+ }
+ }
+ .boxed())
+ }
+}
- // Apply row selection filtering.
- let records: Vec<Value> = match row_selection {
- Some(ref ranges) => {
- let total_rows = all_records.len();
- let mask = ranges_to_mask(total_rows, ranges);
- all_records
+struct AvroReadContext {
+ fields: Vec<DataField>,
+ schema: SchemaRef,
+ predicates: Option<FilePredicates>,
+ batch_size: usize,
+}
+
+/// Release the encoded block after apache-avro copies its payload and marker.
+struct AvroBlockInput {
+ bytes: Bytes,
+}
+
+impl Read for AvroBlockInput {
+ fn read(&mut self, output: &mut [u8]) -> std::io::Result<usize> {
+ let count = output.len().min(self.bytes.len());
+ output[..count].copy_from_slice(&self.bytes[..count]);
+ self.bytes.advance(count);
+ if self.bytes.is_empty() {
+ self.bytes = Bytes::new();
+ }
+ Ok(count)
+ }
+}
+
+struct AvroBatchDecoder {
+ reader: Reader<'static, AvroBlockInput>,
+ position: u64,
+ end: u64,
+ selection: AvroRowSelection,
+ context: Arc<AvroReadContext>,
+}
+
+impl AvroBatchDecoder {
+ fn next_batch(mut self) -> crate::Result<(Self, Option<RecordBatch>)> {
+ let count = (self.end - self.position).min(self.context.batch_size as
u64);
+ let mut records = Vec::with_capacity(count as usize);
+ // Bound physical decoding too: sparse ranges must not turn a single
+ // non-cancellable blocking task into an entire-block record scan.
+ for _ in 0..count {
+ if self.selection.is_finished() {
+ break;
+ }
+ let record = self
+ .reader
+ .next()
+ .ok_or_else(|| avro_invalid("Truncated Avro block"))?
+ .map_err(avro_decode_error)?;
+ if self.selection.contains(self.position) {
+ records.push(record);
+ }
+ self.position += 1;
+ self.selection.advance_to(self.position);
+ }
+ let batch = if records.is_empty() {
+ None
+ } else {
+ let context = &self.context;
+ let batch = records_to_batch(&records, &context.fields,
&context.schema)?;
+ Some(match context.predicates.as_ref() {
+ Some(fp) =>
crate::arrow::residual::filter_record_batch_by_predicates(
+ batch,
+ fp,
+ &context.fields,
+ )?,
+ None => batch,
+ })
+ };
+ Ok((self, batch))
+ }
+}
+
+async fn avro_blocking<T: Send + 'static>(
+ task: impl FnOnce() -> crate::Result<T> + Send + 'static,
+) -> crate::Result<T> {
+ tokio::task::spawn_blocking(task)
+ .await
+ .map_err(|error| Error::UnexpectedError {
+ message: "Avro decoding task failed".to_string(),
+ source: Some(Box::new(error)),
+ })?
+}
+
+fn avro_invalid(message: &str) -> Error {
+ Error::DataInvalid {
+ message: message.to_string(),
+ source: None,
+ }
+}
+
+fn avro_decode_error(error: apache_avro::Error) -> Error {
+ Error::UnexpectedError {
+ message: format!("Failed to decode Avro file: {error}"),
+ source: Some(Box::new(error)),
+ }
+}
+
+fn check_avro_marker(actual: &[u8], expected: &[u8; 16]) -> crate::Result<()> {
+ if actual != expected {
+ return Err(avro_invalid("Avro sync marker mismatch"));
+ }
+ Ok(())
+}
+
+/// Sorted inclusive ranges, consumed in original physical row order.
+struct AvroRowSelection {
+ ranges: Option<VecDeque<RowRange>>,
+}
+
+impl AvroRowSelection {
+ fn new(ranges: Option<Vec<RowRange>>) -> Self {
+ Self {
+ ranges: ranges.map(|ranges| {
+ crate::table::merge_row_ranges(ranges)
.into_iter()
- .enumerate()
- .filter(|(i, _)| mask[*i])
- .map(|(_, r)| r)
+ .filter(|range| range.to() >= 0)
+ .map(|range| RowRange::new(range.from().max(0),
range.to()))
.collect()
+ }),
+ }
+ }
+
+ fn advance_to(&mut self, position: u64) {
+ if let Some(ranges) = &mut self.ranges {
+ while ranges
+ .front()
+ .is_some_and(|range| (range.to() as u64) < position)
+ {
+ ranges.pop_front();
}
- None => all_records,
- };
+ }
+ }
- Ok(try_stream! {
- for chunk in records.chunks(batch_size) {
- let batch = records_to_batch(chunk, &scan_fields,
&target_schema)?;
- let batch = match predicates.as_ref() {
- Some(fp) =>
crate::arrow::residual::filter_record_batch_by_predicates(
- batch,
- fp,
- &scan_fields,
- )?,
- None => batch,
- };
- yield batch;
+ fn is_finished(&self) -> bool {
+ self.ranges.as_ref().is_some_and(VecDeque::is_empty)
+ }
+
+ fn overlaps(&mut self, start: u64, end: u64) -> bool {
+ self.advance_to(start);
+ match &self.ranges {
+ None => true,
+ Some(ranges) => ranges
+ .front()
+ .is_some_and(|range| (range.from() as u64) < end),
+ }
+ }
+
+ fn contains(&mut self, position: u64) -> bool {
+ self.advance_to(position);
+ match &self.ranges {
+ None => true,
+ Some(ranges) => ranges
+ .front()
+ .is_some_and(|range| (range.from() as u64) <= position),
+ }
+ }
+}
+
+/// Small sequential read-ahead buffer. Skipping an OCF payload advances the
+/// offset without fetching it, retaining any already buffered bytes after it.
+struct BufferedAvroInput {
+ reader: Box<dyn FileRead>,
+ file_size: u64,
+ position: u64,
+ buffer: Bytes,
+}
+
+impl BufferedAvroInput {
+ fn new(reader: Box<dyn FileRead>, file_size: u64) -> Self {
+ Self {
+ reader,
+ file_size,
+ position: 0,
+ buffer: Bytes::new(),
+ }
+ }
+
+ fn ensure_available(&self, len: u64) -> crate::Result<()> {
+ if len > self.file_size - self.position {
+ return Err(avro_invalid("Truncated Avro file"));
+ }
+ Ok(())
+ }
+
+ async fn fill(&mut self) -> crate::Result<()> {
+ if self.buffer.is_empty() {
+ self.ensure_available(1)?;
+ let end = self
+ .position
+ .saturating_add(READ_BUFFER_SIZE)
+ .min(self.file_size);
+ self.buffer = self.reader.read(self.position..end).await?;
Review Comment:
Addressed in b7600be6. Reads start with 64 KiB of read-ahead; known block
payloads are fetched in requests of up to 8 MiB. After the first batch,
unrestricted scans grow their read-ahead window to 8 MiB, while explicit row
ranges retain the smaller window.
`full_scan_coalesces_reads_for_the_first_block_and_subsequent_blocks` checks
both the first-block request count and the subsequent full-scan requests,
alongside the existing prefix/drop test. The PR description includes the
earlier before/after measurements and identifies the measured revisions.
##########
crates/paimon/src/arrow/format/avro.rs:
##########
@@ -52,66 +67,381 @@ impl FormatFileReader for AvroFormatReader {
batch_size: Option<usize>,
row_selection: Option<Vec<RowRange>>,
) -> crate::Result<ArrowRecordBatchStream> {
- // NOTE: Avro OCF requires sequential reading, so we load the entire
file into memory.
- // This is fine for typical Paimon data files but may be problematic
for very large files.
- let file_bytes = reader.read(0..file_size).await?;
-
- // Widen the decoded schema to include any predicate columns that are
not
- // in the projection, so residual filtering can see them.
DataFileReader
- // projects the returned batch to `read_fields` by name, dropping
extras.
+ // Residual filters need unprojected predicate columns; DataFileReader
+ // projects the filtered batch back to the requested fields.
let scan_fields =
crate::arrow::residual::widen_scan_fields(read_fields, predicates);
let target_schema = build_target_arrow_schema(&scan_fields)?;
- let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
- // Own the predicates so the returned 'static stream does not borrow
the
- // caller's `&FilePredicates` (FilePredicates is not `Clone`; rebuild
it).
+ let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
let predicates = predicates.map(|fp| FilePredicates {
predicates: fp.predicates.clone(),
row_filter_factory: None,
file_fields: fp.file_fields.clone(),
});
- // Collect Avro records directly as apache_avro::Value, avoiding
intermediate conversion.
- let all_records: Vec<Value> = Reader::new(&file_bytes[..])
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to open Avro file: {e}"),
- source: Some(Box::new(e)),
- })?
- .collect::<std::result::Result<Vec<Value>, _>>()
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to deserialize Avro record: {e}"),
- source: Some(Box::new(e)),
- })?;
+ let context = Arc::new(AvroReadContext {
+ fields: scan_fields,
+ schema: target_schema,
+ predicates,
+ batch_size,
+ });
+ let mut selection = AvroRowSelection::new(row_selection);
+
+ Ok(try_stream! {
+ if selection.is_finished() {
+ return;
+ }
+ let mut input = BufferedAvroInput::new(reader, file_size);
+ let (header, marker) = input.read_header().await?;
+ let header = avro_blocking(move || {
+ // Validate header-only files too; selected blocks use the same
+ // apache-avro schema resolution, logical types and codecs.
+ Reader::new(header.as_slice()).map_err(avro_decode_error)?;
+ Ok(header)
+ }).await?;
+ let mut position = 0_u64;
+ while input.position < file_size && !selection.is_finished() {
+ let mut framing = Vec::with_capacity(20);
+ let count = input.read_length(&mut framing).await?;
+ let size = input.read_length(&mut framing).await?;
+ let end = position.checked_add(count)
+ .ok_or_else(|| avro_invalid("Avro row position
overflow"))?;
+ let block_bytes = size.checked_add(16)
+ .ok_or_else(|| avro_invalid("Avro block size overflow"))?;
+ input.ensure_available(block_bytes)?;
+
+ if count == 0 || !selection.overlaps(position, end) {
+ // Block counts advance physical positions without
decoding.
+ input.skip(size)?;
+ input.read_marker(&marker).await?;
+ } else {
+ let mut encoded = header.clone();
+ encoded.extend_from_slice(&framing);
+ input.read_into(block_bytes, &mut encoded).await?;
+ check_avro_marker(&encoded[encoded.len() -
marker.len()..], &marker)?;
+ let context = context.clone();
+ let (mut decoder, mut batch) = avro_blocking(move || {
+ AvroBatchDecoder {
+ reader: Reader::new(AvroBlockInput { bytes:
encoded.into() })
+ .map_err(avro_decode_error)?,
+ position,
+ end,
+ selection,
+ context,
+ }.next_batch()
+ }).await?;
+ loop {
+ if let Some(batch) = batch.take() {
+ yield batch;
+ }
+ if decoder.position == end ||
decoder.selection.is_finished() {
+ selection = decoder.selection;
+ break;
+ }
+ (decoder, batch) = avro_blocking(move ||
decoder.next_batch()).await?;
+ }
+ }
+ position = end;
+ }
+ }
+ .boxed())
+ }
+}
- // Apply row selection filtering.
- let records: Vec<Value> = match row_selection {
- Some(ref ranges) => {
- let total_rows = all_records.len();
- let mask = ranges_to_mask(total_rows, ranges);
- all_records
+struct AvroReadContext {
+ fields: Vec<DataField>,
+ schema: SchemaRef,
+ predicates: Option<FilePredicates>,
+ batch_size: usize,
+}
+
+/// Release the encoded block after apache-avro copies its payload and marker.
+struct AvroBlockInput {
+ bytes: Bytes,
+}
+
+impl Read for AvroBlockInput {
+ fn read(&mut self, output: &mut [u8]) -> std::io::Result<usize> {
+ let count = output.len().min(self.bytes.len());
+ output[..count].copy_from_slice(&self.bytes[..count]);
+ self.bytes.advance(count);
+ if self.bytes.is_empty() {
+ self.bytes = Bytes::new();
+ }
+ Ok(count)
+ }
+}
+
+struct AvroBatchDecoder {
+ reader: Reader<'static, AvroBlockInput>,
+ position: u64,
+ end: u64,
+ selection: AvroRowSelection,
+ context: Arc<AvroReadContext>,
+}
+
+impl AvroBatchDecoder {
+ fn next_batch(mut self) -> crate::Result<(Self, Option<RecordBatch>)> {
+ let count = (self.end - self.position).min(self.context.batch_size as
u64);
+ let mut records = Vec::with_capacity(count as usize);
+ // Bound physical decoding too: sparse ranges must not turn a single
+ // non-cancellable blocking task into an entire-block record scan.
+ for _ in 0..count {
+ if self.selection.is_finished() {
+ break;
+ }
+ let record = self
+ .reader
+ .next()
+ .ok_or_else(|| avro_invalid("Truncated Avro block"))?
+ .map_err(avro_decode_error)?;
+ if self.selection.contains(self.position) {
+ records.push(record);
+ }
+ self.position += 1;
+ self.selection.advance_to(self.position);
+ }
+ let batch = if records.is_empty() {
+ None
+ } else {
+ let context = &self.context;
+ let batch = records_to_batch(&records, &context.fields,
&context.schema)?;
+ Some(match context.predicates.as_ref() {
+ Some(fp) =>
crate::arrow::residual::filter_record_batch_by_predicates(
+ batch,
+ fp,
+ &context.fields,
+ )?,
+ None => batch,
+ })
+ };
+ Ok((self, batch))
+ }
+}
+
+async fn avro_blocking<T: Send + 'static>(
+ task: impl FnOnce() -> crate::Result<T> + Send + 'static,
+) -> crate::Result<T> {
+ tokio::task::spawn_blocking(task)
Review Comment:
Addressed in b7600be6. `run_decode_task` checks `Handle::try_current()` and
decodes inline when there is no Tokio runtime, yielding between physical
batches. The fallback also converts decoder panics into stream errors. Tests
read a Java Avro fixture through memory-backed FileIO on
`futures::executor::block_on`, cover corrupt payloads on both executors, and
verify cancellation during a sparse selection with fully buffered input.
##########
crates/paimon/src/arrow/format/avro.rs:
##########
@@ -52,66 +67,381 @@ impl FormatFileReader for AvroFormatReader {
batch_size: Option<usize>,
row_selection: Option<Vec<RowRange>>,
) -> crate::Result<ArrowRecordBatchStream> {
- // NOTE: Avro OCF requires sequential reading, so we load the entire
file into memory.
- // This is fine for typical Paimon data files but may be problematic
for very large files.
- let file_bytes = reader.read(0..file_size).await?;
-
- // Widen the decoded schema to include any predicate columns that are
not
- // in the projection, so residual filtering can see them.
DataFileReader
- // projects the returned batch to `read_fields` by name, dropping
extras.
+ // Residual filters need unprojected predicate columns; DataFileReader
+ // projects the filtered batch back to the requested fields.
let scan_fields =
crate::arrow::residual::widen_scan_fields(read_fields, predicates);
let target_schema = build_target_arrow_schema(&scan_fields)?;
- let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
- // Own the predicates so the returned 'static stream does not borrow
the
- // caller's `&FilePredicates` (FilePredicates is not `Clone`; rebuild
it).
+ let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
let predicates = predicates.map(|fp| FilePredicates {
predicates: fp.predicates.clone(),
row_filter_factory: None,
file_fields: fp.file_fields.clone(),
});
- // Collect Avro records directly as apache_avro::Value, avoiding
intermediate conversion.
- let all_records: Vec<Value> = Reader::new(&file_bytes[..])
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to open Avro file: {e}"),
- source: Some(Box::new(e)),
- })?
- .collect::<std::result::Result<Vec<Value>, _>>()
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to deserialize Avro record: {e}"),
- source: Some(Box::new(e)),
- })?;
+ let context = Arc::new(AvroReadContext {
+ fields: scan_fields,
+ schema: target_schema,
+ predicates,
+ batch_size,
+ });
+ let mut selection = AvroRowSelection::new(row_selection);
+
+ Ok(try_stream! {
+ if selection.is_finished() {
+ return;
+ }
+ let mut input = BufferedAvroInput::new(reader, file_size);
+ let (header, marker) = input.read_header().await?;
+ let header = avro_blocking(move || {
+ // Validate header-only files too; selected blocks use the same
+ // apache-avro schema resolution, logical types and codecs.
+ Reader::new(header.as_slice()).map_err(avro_decode_error)?;
+ Ok(header)
+ }).await?;
+ let mut position = 0_u64;
+ while input.position < file_size && !selection.is_finished() {
+ let mut framing = Vec::with_capacity(20);
+ let count = input.read_length(&mut framing).await?;
+ let size = input.read_length(&mut framing).await?;
+ let end = position.checked_add(count)
+ .ok_or_else(|| avro_invalid("Avro row position
overflow"))?;
+ let block_bytes = size.checked_add(16)
+ .ok_or_else(|| avro_invalid("Avro block size overflow"))?;
+ input.ensure_available(block_bytes)?;
+
+ if count == 0 || !selection.overlaps(position, end) {
+ // Block counts advance physical positions without
decoding.
+ input.skip(size)?;
+ input.read_marker(&marker).await?;
+ } else {
+ let mut encoded = header.clone();
+ encoded.extend_from_slice(&framing);
+ input.read_into(block_bytes, &mut encoded).await?;
+ check_avro_marker(&encoded[encoded.len() -
marker.len()..], &marker)?;
+ let context = context.clone();
+ let (mut decoder, mut batch) = avro_blocking(move || {
+ AvroBatchDecoder {
+ reader: Reader::new(AvroBlockInput { bytes:
encoded.into() })
Review Comment:
Addressed in b7600be6. `AvroBatchDecoder` creates one `apache_avro::Reader`
after reading the header and keeps it for the file. `AvroBlockInput` supplies
successive selected blocks to that reader, so the writer schema and resolved
names are reused across blocks. The codec and nested-value tests exercise
multiple blocks, including named record references. The PR description retains
the earlier comparison results with their exact revision and fixture details.
##########
crates/paimon/src/arrow/format/avro.rs:
##########
@@ -52,66 +67,381 @@ impl FormatFileReader for AvroFormatReader {
batch_size: Option<usize>,
row_selection: Option<Vec<RowRange>>,
) -> crate::Result<ArrowRecordBatchStream> {
- // NOTE: Avro OCF requires sequential reading, so we load the entire
file into memory.
- // This is fine for typical Paimon data files but may be problematic
for very large files.
- let file_bytes = reader.read(0..file_size).await?;
-
- // Widen the decoded schema to include any predicate columns that are
not
- // in the projection, so residual filtering can see them.
DataFileReader
- // projects the returned batch to `read_fields` by name, dropping
extras.
+ // Residual filters need unprojected predicate columns; DataFileReader
+ // projects the filtered batch back to the requested fields.
let scan_fields =
crate::arrow::residual::widen_scan_fields(read_fields, predicates);
let target_schema = build_target_arrow_schema(&scan_fields)?;
- let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
- // Own the predicates so the returned 'static stream does not borrow
the
- // caller's `&FilePredicates` (FilePredicates is not `Clone`; rebuild
it).
+ let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
let predicates = predicates.map(|fp| FilePredicates {
predicates: fp.predicates.clone(),
row_filter_factory: None,
file_fields: fp.file_fields.clone(),
});
- // Collect Avro records directly as apache_avro::Value, avoiding
intermediate conversion.
- let all_records: Vec<Value> = Reader::new(&file_bytes[..])
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to open Avro file: {e}"),
- source: Some(Box::new(e)),
- })?
- .collect::<std::result::Result<Vec<Value>, _>>()
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to deserialize Avro record: {e}"),
- source: Some(Box::new(e)),
- })?;
+ let context = Arc::new(AvroReadContext {
+ fields: scan_fields,
+ schema: target_schema,
+ predicates,
+ batch_size,
+ });
+ let mut selection = AvroRowSelection::new(row_selection);
+
+ Ok(try_stream! {
+ if selection.is_finished() {
+ return;
+ }
+ let mut input = BufferedAvroInput::new(reader, file_size);
+ let (header, marker) = input.read_header().await?;
+ let header = avro_blocking(move || {
+ // Validate header-only files too; selected blocks use the same
+ // apache-avro schema resolution, logical types and codecs.
+ Reader::new(header.as_slice()).map_err(avro_decode_error)?;
+ Ok(header)
+ }).await?;
+ let mut position = 0_u64;
+ while input.position < file_size && !selection.is_finished() {
+ let mut framing = Vec::with_capacity(20);
+ let count = input.read_length(&mut framing).await?;
+ let size = input.read_length(&mut framing).await?;
+ let end = position.checked_add(count)
+ .ok_or_else(|| avro_invalid("Avro row position
overflow"))?;
+ let block_bytes = size.checked_add(16)
+ .ok_or_else(|| avro_invalid("Avro block size overflow"))?;
+ input.ensure_available(block_bytes)?;
+
+ if count == 0 || !selection.overlaps(position, end) {
+ // Block counts advance physical positions without
decoding.
+ input.skip(size)?;
+ input.read_marker(&marker).await?;
+ } else {
+ let mut encoded = header.clone();
+ encoded.extend_from_slice(&framing);
+ input.read_into(block_bytes, &mut encoded).await?;
+ check_avro_marker(&encoded[encoded.len() -
marker.len()..], &marker)?;
+ let context = context.clone();
+ let (mut decoder, mut batch) = avro_blocking(move || {
+ AvroBatchDecoder {
+ reader: Reader::new(AvroBlockInput { bytes:
encoded.into() })
+ .map_err(avro_decode_error)?,
+ position,
+ end,
+ selection,
+ context,
+ }.next_batch()
+ }).await?;
+ loop {
+ if let Some(batch) = batch.take() {
+ yield batch;
+ }
+ if decoder.position == end ||
decoder.selection.is_finished() {
+ selection = decoder.selection;
+ break;
+ }
+ (decoder, batch) = avro_blocking(move ||
decoder.next_batch()).await?;
+ }
+ }
+ position = end;
+ }
+ }
+ .boxed())
+ }
+}
- // Apply row selection filtering.
- let records: Vec<Value> = match row_selection {
- Some(ref ranges) => {
- let total_rows = all_records.len();
- let mask = ranges_to_mask(total_rows, ranges);
- all_records
+struct AvroReadContext {
+ fields: Vec<DataField>,
+ schema: SchemaRef,
+ predicates: Option<FilePredicates>,
+ batch_size: usize,
+}
+
+/// Release the encoded block after apache-avro copies its payload and marker.
+struct AvroBlockInput {
+ bytes: Bytes,
+}
+
+impl Read for AvroBlockInput {
+ fn read(&mut self, output: &mut [u8]) -> std::io::Result<usize> {
+ let count = output.len().min(self.bytes.len());
+ output[..count].copy_from_slice(&self.bytes[..count]);
+ self.bytes.advance(count);
+ if self.bytes.is_empty() {
+ self.bytes = Bytes::new();
+ }
+ Ok(count)
+ }
+}
+
+struct AvroBatchDecoder {
+ reader: Reader<'static, AvroBlockInput>,
+ position: u64,
+ end: u64,
+ selection: AvroRowSelection,
+ context: Arc<AvroReadContext>,
+}
+
+impl AvroBatchDecoder {
+ fn next_batch(mut self) -> crate::Result<(Self, Option<RecordBatch>)> {
+ let count = (self.end - self.position).min(self.context.batch_size as
u64);
+ let mut records = Vec::with_capacity(count as usize);
+ // Bound physical decoding too: sparse ranges must not turn a single
+ // non-cancellable blocking task into an entire-block record scan.
+ for _ in 0..count {
+ if self.selection.is_finished() {
+ break;
+ }
+ let record = self
+ .reader
+ .next()
+ .ok_or_else(|| avro_invalid("Truncated Avro block"))?
+ .map_err(avro_decode_error)?;
+ if self.selection.contains(self.position) {
+ records.push(record);
+ }
+ self.position += 1;
+ self.selection.advance_to(self.position);
+ }
+ let batch = if records.is_empty() {
+ None
+ } else {
+ let context = &self.context;
+ let batch = records_to_batch(&records, &context.fields,
&context.schema)?;
+ Some(match context.predicates.as_ref() {
+ Some(fp) =>
crate::arrow::residual::filter_record_batch_by_predicates(
+ batch,
+ fp,
+ &context.fields,
+ )?,
+ None => batch,
+ })
+ };
+ Ok((self, batch))
+ }
+}
+
+async fn avro_blocking<T: Send + 'static>(
+ task: impl FnOnce() -> crate::Result<T> + Send + 'static,
+) -> crate::Result<T> {
+ tokio::task::spawn_blocking(task)
+ .await
Review Comment:
Addressed in b7600be6. The blocking task is wrapped in `AbortOnDropHandle`,
and its queued closure holds only a weak reference to the pending input.
Dropping the waiting future therefore aborts queued work and releases its
captured data without waiting for a blocking worker.
`dropping_decode_future_releases_and_aborts_queued_work` occupies the sole
blocking thread, queues a decode, drops it, verifies the captured state is
released, then frees the worker and confirms the decode never ran. A task that
has already started may finish its current block decompression and physical
batch.
##########
crates/paimon/src/arrow/format/avro.rs:
##########
@@ -52,66 +67,381 @@ impl FormatFileReader for AvroFormatReader {
batch_size: Option<usize>,
row_selection: Option<Vec<RowRange>>,
) -> crate::Result<ArrowRecordBatchStream> {
- // NOTE: Avro OCF requires sequential reading, so we load the entire
file into memory.
- // This is fine for typical Paimon data files but may be problematic
for very large files.
- let file_bytes = reader.read(0..file_size).await?;
-
- // Widen the decoded schema to include any predicate columns that are
not
- // in the projection, so residual filtering can see them.
DataFileReader
- // projects the returned batch to `read_fields` by name, dropping
extras.
+ // Residual filters need unprojected predicate columns; DataFileReader
+ // projects the filtered batch back to the requested fields.
let scan_fields =
crate::arrow::residual::widen_scan_fields(read_fields, predicates);
let target_schema = build_target_arrow_schema(&scan_fields)?;
- let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
- // Own the predicates so the returned 'static stream does not borrow
the
- // caller's `&FilePredicates` (FilePredicates is not `Clone`; rebuild
it).
+ let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
let predicates = predicates.map(|fp| FilePredicates {
predicates: fp.predicates.clone(),
row_filter_factory: None,
file_fields: fp.file_fields.clone(),
});
- // Collect Avro records directly as apache_avro::Value, avoiding
intermediate conversion.
- let all_records: Vec<Value> = Reader::new(&file_bytes[..])
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to open Avro file: {e}"),
- source: Some(Box::new(e)),
- })?
- .collect::<std::result::Result<Vec<Value>, _>>()
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to deserialize Avro record: {e}"),
- source: Some(Box::new(e)),
- })?;
+ let context = Arc::new(AvroReadContext {
+ fields: scan_fields,
+ schema: target_schema,
+ predicates,
+ batch_size,
+ });
+ let mut selection = AvroRowSelection::new(row_selection);
+
+ Ok(try_stream! {
+ if selection.is_finished() {
+ return;
+ }
+ let mut input = BufferedAvroInput::new(reader, file_size);
+ let (header, marker) = input.read_header().await?;
+ let header = avro_blocking(move || {
+ // Validate header-only files too; selected blocks use the same
+ // apache-avro schema resolution, logical types and codecs.
+ Reader::new(header.as_slice()).map_err(avro_decode_error)?;
+ Ok(header)
+ }).await?;
+ let mut position = 0_u64;
+ while input.position < file_size && !selection.is_finished() {
+ let mut framing = Vec::with_capacity(20);
+ let count = input.read_length(&mut framing).await?;
+ let size = input.read_length(&mut framing).await?;
+ let end = position.checked_add(count)
+ .ok_or_else(|| avro_invalid("Avro row position
overflow"))?;
+ let block_bytes = size.checked_add(16)
+ .ok_or_else(|| avro_invalid("Avro block size overflow"))?;
+ input.ensure_available(block_bytes)?;
+
+ if count == 0 || !selection.overlaps(position, end) {
+ // Block counts advance physical positions without
decoding.
+ input.skip(size)?;
+ input.read_marker(&marker).await?;
+ } else {
+ let mut encoded = header.clone();
+ encoded.extend_from_slice(&framing);
+ input.read_into(block_bytes, &mut encoded).await?;
+ check_avro_marker(&encoded[encoded.len() -
marker.len()..], &marker)?;
Review Comment:
Addressed in b7600be6. The scan now uses an owning `try_unfold` step, so
read/decode state is dropped before a terminal error is yielded. The shared
`terminal_error` assertion retains the errored stream and verifies both that
the FileRead was dropped and that its storage window has no remaining stream
owner, before polling again. This covers marker mismatches, malformed/truncated
input, codec errors and injected I/O failures.
##########
crates/paimon/src/arrow/format/avro/streaming_tests/benchmark.rs:
##########
@@ -0,0 +1,185 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::*;
+use std::path::PathBuf;
+use std::sync::atomic::AtomicU64;
+use std::time::Instant;
+use tokio::io::{AsyncReadExt, AsyncSeekExt};
+
+const BLOCK_SIZE: usize = 1024 * 1024;
+const BATCH_SIZE: usize = 1024;
+const ROWS: usize = 2_000_000;
+const PAYLOAD_SIZE: usize = 128;
+
+struct BenchmarkRead {
+ file: tokio::sync::Mutex<tokio::fs::File>,
+ bytes_read: Arc<AtomicU64>,
+}
+
+#[async_trait]
+impl FileRead for BenchmarkRead {
+ async fn read(&self, range: Range<u64>) -> crate::Result<Bytes> {
+ let mut file = self.file.lock().await;
+ let mut buffer = vec![0; (range.end - range.start) as usize];
+ file.seek(std::io::SeekFrom::Start(range.start))
+ .await
+ .map_err(io_error)?;
+ file.read_exact(&mut buffer).await.map_err(io_error)?;
+ self.bytes_read
+ .fetch_add(buffer.len() as u64, Ordering::Relaxed);
+ Ok(buffer.into())
+ }
+}
+
+fn io_error(error: std::io::Error) -> Error {
+ Error::UnexpectedError {
+ message: "Avro benchmark I/O failed".into(),
+ source: Some(Box::new(error)),
+ }
+}
+
+async fn eager_stream(
+ reader: Box<dyn FileRead>,
+ size: u64,
+) -> crate::Result<ArrowRecordBatchStream> {
+ let bytes = reader.read(0..size).await?;
+ let records = Reader::new(bytes.as_ref())
+ .map_err(avro_decode_error)?
+ .collect::<std::result::Result<Vec<_>, _>>()
+ .map_err(avro_decode_error)?;
+ let fields = fields();
+ let schema = build_target_arrow_schema(&fields)?;
+ Ok(try_stream! {
+ for records in records.chunks(BATCH_SIZE) {
+ yield records_to_batch(records, &fields, &schema)?;
+ }
+ }
+ .boxed())
+}
+
+fn peak_rss_kib() -> Option<u64> {
+ std::fs::read_to_string("/proc/self/status")
+ .ok()?
+ .lines()
+ .find_map(|line| {
+ line.strip_prefix("VmHWM:")?
+ .split_whitespace()
+ .next()?
+ .parse()
+ .ok()
+ })
+}
+
+#[tokio::test]
+#[ignore = "manual Avro full-scan/LIMIT comparison; see
benchmarks/avro/README.md"]
+async fn compare_avro_reads() {
Review Comment:
Addressed in b7600be6. Removed `streaming_tests/benchmark.rs` and its module
declaration. The README now documents the streaming behavior and links to the
PR description, which contains the existing full-scan and LIMIT measurements,
fixture recipe, cache policy and measurement method. Those figures remain
explicitly attributed to the earlier measured revisions; the latest head's 83
Avro tests, full unit suite, formatting and Clippy results are reported
separately.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]