JingsongLi commented on code in PR #1449:
URL: https://github.com/apache/paimon-rust/pull/1449#discussion_r4611067961
##########
crates/paimon/src/arrow/format/avro.rs:
##########
@@ -52,66 +67,381 @@ impl FormatFileReader for AvroFormatReader {
batch_size: Option<usize>,
row_selection: Option<Vec<RowRange>>,
) -> crate::Result<ArrowRecordBatchStream> {
- // NOTE: Avro OCF requires sequential reading, so we load the entire
file into memory.
- // This is fine for typical Paimon data files but may be problematic
for very large files.
- let file_bytes = reader.read(0..file_size).await?;
-
- // Widen the decoded schema to include any predicate columns that are
not
- // in the projection, so residual filtering can see them.
DataFileReader
- // projects the returned batch to `read_fields` by name, dropping
extras.
+ // Residual filters need unprojected predicate columns; DataFileReader
+ // projects the filtered batch back to the requested fields.
let scan_fields =
crate::arrow::residual::widen_scan_fields(read_fields, predicates);
let target_schema = build_target_arrow_schema(&scan_fields)?;
- let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
- // Own the predicates so the returned 'static stream does not borrow
the
- // caller's `&FilePredicates` (FilePredicates is not `Clone`; rebuild
it).
+ let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
let predicates = predicates.map(|fp| FilePredicates {
predicates: fp.predicates.clone(),
row_filter_factory: None,
file_fields: fp.file_fields.clone(),
});
- // Collect Avro records directly as apache_avro::Value, avoiding
intermediate conversion.
- let all_records: Vec<Value> = Reader::new(&file_bytes[..])
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to open Avro file: {e}"),
- source: Some(Box::new(e)),
- })?
- .collect::<std::result::Result<Vec<Value>, _>>()
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to deserialize Avro record: {e}"),
- source: Some(Box::new(e)),
- })?;
+ let context = Arc::new(AvroReadContext {
+ fields: scan_fields,
+ schema: target_schema,
+ predicates,
+ batch_size,
+ });
+ let mut selection = AvroRowSelection::new(row_selection);
+
+ Ok(try_stream! {
+ if selection.is_finished() {
+ return;
+ }
+ let mut input = BufferedAvroInput::new(reader, file_size);
+ let (header, marker) = input.read_header().await?;
+ let header = avro_blocking(move || {
+ // Validate header-only files too; selected blocks use the same
+ // apache-avro schema resolution, logical types and codecs.
+ Reader::new(header.as_slice()).map_err(avro_decode_error)?;
+ Ok(header)
+ }).await?;
+ let mut position = 0_u64;
+ while input.position < file_size && !selection.is_finished() {
+ let mut framing = Vec::with_capacity(20);
+ let count = input.read_length(&mut framing).await?;
+ let size = input.read_length(&mut framing).await?;
+ let end = position.checked_add(count)
+ .ok_or_else(|| avro_invalid("Avro row position
overflow"))?;
+ let block_bytes = size.checked_add(16)
+ .ok_or_else(|| avro_invalid("Avro block size overflow"))?;
+ input.ensure_available(block_bytes)?;
+
+ if count == 0 || !selection.overlaps(position, end) {
+ // Block counts advance physical positions without
decoding.
+ input.skip(size)?;
+ input.read_marker(&marker).await?;
+ } else {
+ let mut encoded = header.clone();
+ encoded.extend_from_slice(&framing);
+ input.read_into(block_bytes, &mut encoded).await?;
+ check_avro_marker(&encoded[encoded.len() -
marker.len()..], &marker)?;
+ let context = context.clone();
+ let (mut decoder, mut batch) = avro_blocking(move || {
+ AvroBatchDecoder {
+ reader: Reader::new(AvroBlockInput { bytes:
encoded.into() })
Review Comment:
`Reader::new` parses the writer schema and resolves its names again for
every selected block. I ran the comparison helper on a 16 MiB, 98-column
fixture with roughly 64 KiB blocks, the Rust writer default. Median full-scan
time over three local debug runs went from 84 ms (eager) to 346 ms (streaming).
With the same rows in 1 MiB blocks it was 83 ms and 135 ms respectively. Could
we initialize the reader once per file and feed it successive selected blocks?
##########
crates/paimon/src/arrow/format/avro.rs:
##########
@@ -52,66 +67,381 @@ impl FormatFileReader for AvroFormatReader {
batch_size: Option<usize>,
row_selection: Option<Vec<RowRange>>,
) -> crate::Result<ArrowRecordBatchStream> {
- // NOTE: Avro OCF requires sequential reading, so we load the entire
file into memory.
- // This is fine for typical Paimon data files but may be problematic
for very large files.
- let file_bytes = reader.read(0..file_size).await?;
-
- // Widen the decoded schema to include any predicate columns that are
not
- // in the projection, so residual filtering can see them.
DataFileReader
- // projects the returned batch to `read_fields` by name, dropping
extras.
+ // Residual filters need unprojected predicate columns; DataFileReader
+ // projects the filtered batch back to the requested fields.
let scan_fields =
crate::arrow::residual::widen_scan_fields(read_fields, predicates);
let target_schema = build_target_arrow_schema(&scan_fields)?;
- let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
- // Own the predicates so the returned 'static stream does not borrow
the
- // caller's `&FilePredicates` (FilePredicates is not `Clone`; rebuild
it).
+ let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
let predicates = predicates.map(|fp| FilePredicates {
predicates: fp.predicates.clone(),
row_filter_factory: None,
file_fields: fp.file_fields.clone(),
});
- // Collect Avro records directly as apache_avro::Value, avoiding
intermediate conversion.
- let all_records: Vec<Value> = Reader::new(&file_bytes[..])
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to open Avro file: {e}"),
- source: Some(Box::new(e)),
- })?
- .collect::<std::result::Result<Vec<Value>, _>>()
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to deserialize Avro record: {e}"),
- source: Some(Box::new(e)),
- })?;
+ let context = Arc::new(AvroReadContext {
+ fields: scan_fields,
+ schema: target_schema,
+ predicates,
+ batch_size,
+ });
+ let mut selection = AvroRowSelection::new(row_selection);
+
+ Ok(try_stream! {
+ if selection.is_finished() {
+ return;
+ }
+ let mut input = BufferedAvroInput::new(reader, file_size);
+ let (header, marker) = input.read_header().await?;
+ let header = avro_blocking(move || {
+ // Validate header-only files too; selected blocks use the same
+ // apache-avro schema resolution, logical types and codecs.
+ Reader::new(header.as_slice()).map_err(avro_decode_error)?;
+ Ok(header)
+ }).await?;
+ let mut position = 0_u64;
+ while input.position < file_size && !selection.is_finished() {
+ let mut framing = Vec::with_capacity(20);
+ let count = input.read_length(&mut framing).await?;
+ let size = input.read_length(&mut framing).await?;
+ let end = position.checked_add(count)
+ .ok_or_else(|| avro_invalid("Avro row position
overflow"))?;
+ let block_bytes = size.checked_add(16)
+ .ok_or_else(|| avro_invalid("Avro block size overflow"))?;
+ input.ensure_available(block_bytes)?;
+
+ if count == 0 || !selection.overlaps(position, end) {
+ // Block counts advance physical positions without
decoding.
+ input.skip(size)?;
+ input.read_marker(&marker).await?;
+ } else {
+ let mut encoded = header.clone();
+ encoded.extend_from_slice(&framing);
+ input.read_into(block_bytes, &mut encoded).await?;
+ check_avro_marker(&encoded[encoded.len() -
marker.len()..], &marker)?;
+ let context = context.clone();
+ let (mut decoder, mut batch) = avro_blocking(move || {
+ AvroBatchDecoder {
+ reader: Reader::new(AvroBlockInput { bytes:
encoded.into() })
+ .map_err(avro_decode_error)?,
+ position,
+ end,
+ selection,
+ context,
+ }.next_batch()
+ }).await?;
+ loop {
+ if let Some(batch) = batch.take() {
+ yield batch;
+ }
+ if decoder.position == end ||
decoder.selection.is_finished() {
+ selection = decoder.selection;
+ break;
+ }
+ (decoder, batch) = avro_blocking(move ||
decoder.next_batch()).await?;
+ }
+ }
+ position = end;
+ }
+ }
+ .boxed())
+ }
+}
- // Apply row selection filtering.
- let records: Vec<Value> = match row_selection {
- Some(ref ranges) => {
- let total_rows = all_records.len();
- let mask = ranges_to_mask(total_rows, ranges);
- all_records
+struct AvroReadContext {
+ fields: Vec<DataField>,
+ schema: SchemaRef,
+ predicates: Option<FilePredicates>,
+ batch_size: usize,
+}
+
+/// Release the encoded block after apache-avro copies its payload and marker.
+struct AvroBlockInput {
+ bytes: Bytes,
+}
+
+impl Read for AvroBlockInput {
+ fn read(&mut self, output: &mut [u8]) -> std::io::Result<usize> {
+ let count = output.len().min(self.bytes.len());
+ output[..count].copy_from_slice(&self.bytes[..count]);
+ self.bytes.advance(count);
+ if self.bytes.is_empty() {
+ self.bytes = Bytes::new();
+ }
+ Ok(count)
+ }
+}
+
+struct AvroBatchDecoder {
+ reader: Reader<'static, AvroBlockInput>,
+ position: u64,
+ end: u64,
+ selection: AvroRowSelection,
+ context: Arc<AvroReadContext>,
+}
+
+impl AvroBatchDecoder {
+ fn next_batch(mut self) -> crate::Result<(Self, Option<RecordBatch>)> {
+ let count = (self.end - self.position).min(self.context.batch_size as
u64);
+ let mut records = Vec::with_capacity(count as usize);
+ // Bound physical decoding too: sparse ranges must not turn a single
+ // non-cancellable blocking task into an entire-block record scan.
+ for _ in 0..count {
+ if self.selection.is_finished() {
+ break;
+ }
+ let record = self
+ .reader
+ .next()
+ .ok_or_else(|| avro_invalid("Truncated Avro block"))?
+ .map_err(avro_decode_error)?;
+ if self.selection.contains(self.position) {
+ records.push(record);
+ }
+ self.position += 1;
+ self.selection.advance_to(self.position);
+ }
+ let batch = if records.is_empty() {
+ None
+ } else {
+ let context = &self.context;
+ let batch = records_to_batch(&records, &context.fields,
&context.schema)?;
+ Some(match context.predicates.as_ref() {
+ Some(fp) =>
crate::arrow::residual::filter_record_batch_by_predicates(
+ batch,
+ fp,
+ &context.fields,
+ )?,
+ None => batch,
+ })
+ };
+ Ok((self, batch))
+ }
+}
+
+async fn avro_blocking<T: Send + 'static>(
+ task: impl FnOnce() -> crate::Result<T> + Send + 'static,
+) -> crate::Result<T> {
+ tokio::task::spawn_blocking(task)
+ .await
+ .map_err(|error| Error::UnexpectedError {
+ message: "Avro decoding task failed".to_string(),
+ source: Some(Box::new(error)),
+ })?
+}
+
+fn avro_invalid(message: &str) -> Error {
+ Error::DataInvalid {
+ message: message.to_string(),
+ source: None,
+ }
+}
+
+fn avro_decode_error(error: apache_avro::Error) -> Error {
+ Error::UnexpectedError {
+ message: format!("Failed to decode Avro file: {error}"),
+ source: Some(Box::new(error)),
+ }
+}
+
+fn check_avro_marker(actual: &[u8], expected: &[u8; 16]) -> crate::Result<()> {
+ if actual != expected {
+ return Err(avro_invalid("Avro sync marker mismatch"));
+ }
+ Ok(())
+}
+
+/// Sorted inclusive ranges, consumed in original physical row order.
+struct AvroRowSelection {
+ ranges: Option<VecDeque<RowRange>>,
+}
+
+impl AvroRowSelection {
+ fn new(ranges: Option<Vec<RowRange>>) -> Self {
+ Self {
+ ranges: ranges.map(|ranges| {
+ crate::table::merge_row_ranges(ranges)
.into_iter()
- .enumerate()
- .filter(|(i, _)| mask[*i])
- .map(|(_, r)| r)
+ .filter(|range| range.to() >= 0)
+ .map(|range| RowRange::new(range.from().max(0),
range.to()))
.collect()
+ }),
+ }
+ }
+
+ fn advance_to(&mut self, position: u64) {
+ if let Some(ranges) = &mut self.ranges {
+ while ranges
+ .front()
+ .is_some_and(|range| (range.to() as u64) < position)
+ {
+ ranges.pop_front();
}
- None => all_records,
- };
+ }
+ }
- Ok(try_stream! {
- for chunk in records.chunks(batch_size) {
- let batch = records_to_batch(chunk, &scan_fields,
&target_schema)?;
- let batch = match predicates.as_ref() {
- Some(fp) =>
crate::arrow::residual::filter_record_batch_by_predicates(
- batch,
- fp,
- &scan_fields,
- )?,
- None => batch,
- };
- yield batch;
+ fn is_finished(&self) -> bool {
+ self.ranges.as_ref().is_some_and(VecDeque::is_empty)
+ }
+
+ fn overlaps(&mut self, start: u64, end: u64) -> bool {
+ self.advance_to(start);
+ match &self.ranges {
+ None => true,
+ Some(ranges) => ranges
+ .front()
+ .is_some_and(|range| (range.from() as u64) < end),
+ }
+ }
+
+ fn contains(&mut self, position: u64) -> bool {
+ self.advance_to(position);
+ match &self.ranges {
+ None => true,
+ Some(ranges) => ranges
+ .front()
+ .is_some_and(|range| (range.from() as u64) <= position),
+ }
+ }
+}
+
+/// Small sequential read-ahead buffer. Skipping an OCF payload advances the
+/// offset without fetching it, retaining any already buffered bytes after it.
+struct BufferedAvroInput {
+ reader: Box<dyn FileRead>,
+ file_size: u64,
+ position: u64,
+ buffer: Bytes,
+}
+
+impl BufferedAvroInput {
+ fn new(reader: Box<dyn FileRead>, file_size: u64) -> Self {
+ Self {
+ reader,
+ file_size,
+ position: 0,
+ buffer: Bytes::new(),
+ }
+ }
+
+ fn ensure_available(&self, len: u64) -> crate::Result<()> {
+ if len > self.file_size - self.position {
+ return Err(avro_invalid("Truncated Avro file"));
+ }
+ Ok(())
+ }
+
+ async fn fill(&mut self) -> crate::Result<()> {
+ if self.buffer.is_empty() {
+ self.ensure_available(1)?;
+ let end = self
+ .position
+ .saturating_add(READ_BUFFER_SIZE)
+ .min(self.file_size);
+ self.buffer = self.reader.read(self.position..end).await?;
Review Comment:
On the default uncached S3 path, each call to `FileRead::read` opens a new
range GET, and this loop waits for it before issuing the next one. A full scan
of a 256 MiB file goes from one GET to 4,096 sequential GETs. Could we keep a
sequential response open and consume it in bounded chunks, or grow the read
window after the first batch? A full-scan test that counts storage requests
would catch this alongside the existing prefix-read test.
##########
crates/paimon/src/arrow/format/avro.rs:
##########
@@ -52,66 +67,381 @@ impl FormatFileReader for AvroFormatReader {
batch_size: Option<usize>,
row_selection: Option<Vec<RowRange>>,
) -> crate::Result<ArrowRecordBatchStream> {
- // NOTE: Avro OCF requires sequential reading, so we load the entire
file into memory.
- // This is fine for typical Paimon data files but may be problematic
for very large files.
- let file_bytes = reader.read(0..file_size).await?;
-
- // Widen the decoded schema to include any predicate columns that are
not
- // in the projection, so residual filtering can see them.
DataFileReader
- // projects the returned batch to `read_fields` by name, dropping
extras.
+ // Residual filters need unprojected predicate columns; DataFileReader
+ // projects the filtered batch back to the requested fields.
let scan_fields =
crate::arrow::residual::widen_scan_fields(read_fields, predicates);
let target_schema = build_target_arrow_schema(&scan_fields)?;
- let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
- // Own the predicates so the returned 'static stream does not borrow
the
- // caller's `&FilePredicates` (FilePredicates is not `Clone`; rebuild
it).
+ let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
let predicates = predicates.map(|fp| FilePredicates {
predicates: fp.predicates.clone(),
row_filter_factory: None,
file_fields: fp.file_fields.clone(),
});
- // Collect Avro records directly as apache_avro::Value, avoiding
intermediate conversion.
- let all_records: Vec<Value> = Reader::new(&file_bytes[..])
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to open Avro file: {e}"),
- source: Some(Box::new(e)),
- })?
- .collect::<std::result::Result<Vec<Value>, _>>()
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to deserialize Avro record: {e}"),
- source: Some(Box::new(e)),
- })?;
+ let context = Arc::new(AvroReadContext {
+ fields: scan_fields,
+ schema: target_schema,
+ predicates,
+ batch_size,
+ });
+ let mut selection = AvroRowSelection::new(row_selection);
+
+ Ok(try_stream! {
+ if selection.is_finished() {
+ return;
+ }
+ let mut input = BufferedAvroInput::new(reader, file_size);
+ let (header, marker) = input.read_header().await?;
+ let header = avro_blocking(move || {
+ // Validate header-only files too; selected blocks use the same
+ // apache-avro schema resolution, logical types and codecs.
+ Reader::new(header.as_slice()).map_err(avro_decode_error)?;
+ Ok(header)
+ }).await?;
+ let mut position = 0_u64;
+ while input.position < file_size && !selection.is_finished() {
+ let mut framing = Vec::with_capacity(20);
+ let count = input.read_length(&mut framing).await?;
+ let size = input.read_length(&mut framing).await?;
+ let end = position.checked_add(count)
+ .ok_or_else(|| avro_invalid("Avro row position
overflow"))?;
+ let block_bytes = size.checked_add(16)
+ .ok_or_else(|| avro_invalid("Avro block size overflow"))?;
+ input.ensure_available(block_bytes)?;
+
+ if count == 0 || !selection.overlaps(position, end) {
+ // Block counts advance physical positions without
decoding.
+ input.skip(size)?;
+ input.read_marker(&marker).await?;
+ } else {
+ let mut encoded = header.clone();
+ encoded.extend_from_slice(&framing);
+ input.read_into(block_bytes, &mut encoded).await?;
+ check_avro_marker(&encoded[encoded.len() -
marker.len()..], &marker)?;
Review Comment:
A marker mismatch here yields `Err` from `try_stream!` while `encoded` and
`input` are still in scope. I confirmed the `FileRead` was still alive after
`try_next().await` returned the error. A format-table caller that retains the
errored stream also retains the block buffer until it polls again or drops the
stream. Could we release the read state before yielding a terminal error and
assert that the reader has been released while the errored stream is still
alive?
##########
crates/paimon/src/arrow/format/avro.rs:
##########
@@ -52,66 +67,381 @@ impl FormatFileReader for AvroFormatReader {
batch_size: Option<usize>,
row_selection: Option<Vec<RowRange>>,
) -> crate::Result<ArrowRecordBatchStream> {
- // NOTE: Avro OCF requires sequential reading, so we load the entire
file into memory.
- // This is fine for typical Paimon data files but may be problematic
for very large files.
- let file_bytes = reader.read(0..file_size).await?;
-
- // Widen the decoded schema to include any predicate columns that are
not
- // in the projection, so residual filtering can see them.
DataFileReader
- // projects the returned batch to `read_fields` by name, dropping
extras.
+ // Residual filters need unprojected predicate columns; DataFileReader
+ // projects the filtered batch back to the requested fields.
let scan_fields =
crate::arrow::residual::widen_scan_fields(read_fields, predicates);
let target_schema = build_target_arrow_schema(&scan_fields)?;
- let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
- // Own the predicates so the returned 'static stream does not borrow
the
- // caller's `&FilePredicates` (FilePredicates is not `Clone`; rebuild
it).
+ let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
let predicates = predicates.map(|fp| FilePredicates {
predicates: fp.predicates.clone(),
row_filter_factory: None,
file_fields: fp.file_fields.clone(),
});
- // Collect Avro records directly as apache_avro::Value, avoiding
intermediate conversion.
- let all_records: Vec<Value> = Reader::new(&file_bytes[..])
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to open Avro file: {e}"),
- source: Some(Box::new(e)),
- })?
- .collect::<std::result::Result<Vec<Value>, _>>()
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to deserialize Avro record: {e}"),
- source: Some(Box::new(e)),
- })?;
+ let context = Arc::new(AvroReadContext {
+ fields: scan_fields,
+ schema: target_schema,
+ predicates,
+ batch_size,
+ });
+ let mut selection = AvroRowSelection::new(row_selection);
+
+ Ok(try_stream! {
+ if selection.is_finished() {
+ return;
+ }
+ let mut input = BufferedAvroInput::new(reader, file_size);
+ let (header, marker) = input.read_header().await?;
+ let header = avro_blocking(move || {
+ // Validate header-only files too; selected blocks use the same
+ // apache-avro schema resolution, logical types and codecs.
+ Reader::new(header.as_slice()).map_err(avro_decode_error)?;
+ Ok(header)
+ }).await?;
+ let mut position = 0_u64;
+ while input.position < file_size && !selection.is_finished() {
+ let mut framing = Vec::with_capacity(20);
+ let count = input.read_length(&mut framing).await?;
+ let size = input.read_length(&mut framing).await?;
+ let end = position.checked_add(count)
+ .ok_or_else(|| avro_invalid("Avro row position
overflow"))?;
+ let block_bytes = size.checked_add(16)
+ .ok_or_else(|| avro_invalid("Avro block size overflow"))?;
+ input.ensure_available(block_bytes)?;
+
+ if count == 0 || !selection.overlaps(position, end) {
+ // Block counts advance physical positions without
decoding.
+ input.skip(size)?;
+ input.read_marker(&marker).await?;
+ } else {
+ let mut encoded = header.clone();
+ encoded.extend_from_slice(&framing);
+ input.read_into(block_bytes, &mut encoded).await?;
+ check_avro_marker(&encoded[encoded.len() -
marker.len()..], &marker)?;
+ let context = context.clone();
+ let (mut decoder, mut batch) = avro_blocking(move || {
+ AvroBatchDecoder {
+ reader: Reader::new(AvroBlockInput { bytes:
encoded.into() })
+ .map_err(avro_decode_error)?,
+ position,
+ end,
+ selection,
+ context,
+ }.next_batch()
+ }).await?;
+ loop {
+ if let Some(batch) = batch.take() {
+ yield batch;
+ }
+ if decoder.position == end ||
decoder.selection.is_finished() {
+ selection = decoder.selection;
+ break;
+ }
+ (decoder, batch) = avro_blocking(move ||
decoder.next_batch()).await?;
+ }
+ }
+ position = end;
+ }
+ }
+ .boxed())
+ }
+}
- // Apply row selection filtering.
- let records: Vec<Value> = match row_selection {
- Some(ref ranges) => {
- let total_rows = all_records.len();
- let mask = ranges_to_mask(total_rows, ranges);
- all_records
+struct AvroReadContext {
+ fields: Vec<DataField>,
+ schema: SchemaRef,
+ predicates: Option<FilePredicates>,
+ batch_size: usize,
+}
+
+/// Release the encoded block after apache-avro copies its payload and marker.
+struct AvroBlockInput {
+ bytes: Bytes,
+}
+
+impl Read for AvroBlockInput {
+ fn read(&mut self, output: &mut [u8]) -> std::io::Result<usize> {
+ let count = output.len().min(self.bytes.len());
+ output[..count].copy_from_slice(&self.bytes[..count]);
+ self.bytes.advance(count);
+ if self.bytes.is_empty() {
+ self.bytes = Bytes::new();
+ }
+ Ok(count)
+ }
+}
+
+struct AvroBatchDecoder {
+ reader: Reader<'static, AvroBlockInput>,
+ position: u64,
+ end: u64,
+ selection: AvroRowSelection,
+ context: Arc<AvroReadContext>,
+}
+
+impl AvroBatchDecoder {
+ fn next_batch(mut self) -> crate::Result<(Self, Option<RecordBatch>)> {
+ let count = (self.end - self.position).min(self.context.batch_size as
u64);
+ let mut records = Vec::with_capacity(count as usize);
+ // Bound physical decoding too: sparse ranges must not turn a single
+ // non-cancellable blocking task into an entire-block record scan.
+ for _ in 0..count {
+ if self.selection.is_finished() {
+ break;
+ }
+ let record = self
+ .reader
+ .next()
+ .ok_or_else(|| avro_invalid("Truncated Avro block"))?
+ .map_err(avro_decode_error)?;
+ if self.selection.contains(self.position) {
+ records.push(record);
+ }
+ self.position += 1;
+ self.selection.advance_to(self.position);
+ }
+ let batch = if records.is_empty() {
+ None
+ } else {
+ let context = &self.context;
+ let batch = records_to_batch(&records, &context.fields,
&context.schema)?;
+ Some(match context.predicates.as_ref() {
+ Some(fp) =>
crate::arrow::residual::filter_record_batch_by_predicates(
+ batch,
+ fp,
+ &context.fields,
+ )?,
+ None => batch,
+ })
+ };
+ Ok((self, batch))
+ }
+}
+
+async fn avro_blocking<T: Send + 'static>(
+ task: impl FnOnce() -> crate::Result<T> + Send + 'static,
+) -> crate::Result<T> {
+ tokio::task::spawn_blocking(task)
+ .await
Review Comment:
Dropping this await detaches the blocking task, including when it is still
queued. With the only blocking thread occupied, I queued a decode, dropped the
waiting future, and released the thread. The decode then ran after
cancellation. Under load, canceled scans can keep their blocks queued and delay
live scans. Could we abort queued work on drop or have the closure check for
cancellation before it starts decoding?
##########
crates/paimon/src/arrow/format/avro.rs:
##########
@@ -52,66 +67,381 @@ impl FormatFileReader for AvroFormatReader {
batch_size: Option<usize>,
row_selection: Option<Vec<RowRange>>,
) -> crate::Result<ArrowRecordBatchStream> {
- // NOTE: Avro OCF requires sequential reading, so we load the entire
file into memory.
- // This is fine for typical Paimon data files but may be problematic
for very large files.
- let file_bytes = reader.read(0..file_size).await?;
-
- // Widen the decoded schema to include any predicate columns that are
not
- // in the projection, so residual filtering can see them.
DataFileReader
- // projects the returned batch to `read_fields` by name, dropping
extras.
+ // Residual filters need unprojected predicate columns; DataFileReader
+ // projects the filtered batch back to the requested fields.
let scan_fields =
crate::arrow::residual::widen_scan_fields(read_fields, predicates);
let target_schema = build_target_arrow_schema(&scan_fields)?;
- let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
- // Own the predicates so the returned 'static stream does not borrow
the
- // caller's `&FilePredicates` (FilePredicates is not `Clone`; rebuild
it).
+ let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
let predicates = predicates.map(|fp| FilePredicates {
predicates: fp.predicates.clone(),
row_filter_factory: None,
file_fields: fp.file_fields.clone(),
});
- // Collect Avro records directly as apache_avro::Value, avoiding
intermediate conversion.
- let all_records: Vec<Value> = Reader::new(&file_bytes[..])
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to open Avro file: {e}"),
- source: Some(Box::new(e)),
- })?
- .collect::<std::result::Result<Vec<Value>, _>>()
- .map_err(|e| Error::UnexpectedError {
- message: format!("Failed to deserialize Avro record: {e}"),
- source: Some(Box::new(e)),
- })?;
+ let context = Arc::new(AvroReadContext {
+ fields: scan_fields,
+ schema: target_schema,
+ predicates,
+ batch_size,
+ });
+ let mut selection = AvroRowSelection::new(row_selection);
+
+ Ok(try_stream! {
+ if selection.is_finished() {
+ return;
+ }
+ let mut input = BufferedAvroInput::new(reader, file_size);
+ let (header, marker) = input.read_header().await?;
+ let header = avro_blocking(move || {
+ // Validate header-only files too; selected blocks use the same
+ // apache-avro schema resolution, logical types and codecs.
+ Reader::new(header.as_slice()).map_err(avro_decode_error)?;
+ Ok(header)
+ }).await?;
+ let mut position = 0_u64;
+ while input.position < file_size && !selection.is_finished() {
+ let mut framing = Vec::with_capacity(20);
+ let count = input.read_length(&mut framing).await?;
+ let size = input.read_length(&mut framing).await?;
+ let end = position.checked_add(count)
+ .ok_or_else(|| avro_invalid("Avro row position
overflow"))?;
+ let block_bytes = size.checked_add(16)
+ .ok_or_else(|| avro_invalid("Avro block size overflow"))?;
+ input.ensure_available(block_bytes)?;
+
+ if count == 0 || !selection.overlaps(position, end) {
+ // Block counts advance physical positions without
decoding.
+ input.skip(size)?;
+ input.read_marker(&marker).await?;
+ } else {
+ let mut encoded = header.clone();
+ encoded.extend_from_slice(&framing);
+ input.read_into(block_bytes, &mut encoded).await?;
+ check_avro_marker(&encoded[encoded.len() -
marker.len()..], &marker)?;
+ let context = context.clone();
+ let (mut decoder, mut batch) = avro_blocking(move || {
+ AvroBatchDecoder {
+ reader: Reader::new(AvroBlockInput { bytes:
encoded.into() })
+ .map_err(avro_decode_error)?,
+ position,
+ end,
+ selection,
+ context,
+ }.next_batch()
+ }).await?;
+ loop {
+ if let Some(batch) = batch.take() {
+ yield batch;
+ }
+ if decoder.position == end ||
decoder.selection.is_finished() {
+ selection = decoder.selection;
+ break;
+ }
+ (decoder, batch) = avro_blocking(move ||
decoder.next_batch()).await?;
+ }
+ }
+ position = end;
+ }
+ }
+ .boxed())
+ }
+}
- // Apply row selection filtering.
- let records: Vec<Value> = match row_selection {
- Some(ref ranges) => {
- let total_rows = all_records.len();
- let mask = ranges_to_mask(total_rows, ranges);
- all_records
+struct AvroReadContext {
+ fields: Vec<DataField>,
+ schema: SchemaRef,
+ predicates: Option<FilePredicates>,
+ batch_size: usize,
+}
+
+/// Release the encoded block after apache-avro copies its payload and marker.
+struct AvroBlockInput {
+ bytes: Bytes,
+}
+
+impl Read for AvroBlockInput {
+ fn read(&mut self, output: &mut [u8]) -> std::io::Result<usize> {
+ let count = output.len().min(self.bytes.len());
+ output[..count].copy_from_slice(&self.bytes[..count]);
+ self.bytes.advance(count);
+ if self.bytes.is_empty() {
+ self.bytes = Bytes::new();
+ }
+ Ok(count)
+ }
+}
+
+struct AvroBatchDecoder {
+ reader: Reader<'static, AvroBlockInput>,
+ position: u64,
+ end: u64,
+ selection: AvroRowSelection,
+ context: Arc<AvroReadContext>,
+}
+
+impl AvroBatchDecoder {
+ fn next_batch(mut self) -> crate::Result<(Self, Option<RecordBatch>)> {
+ let count = (self.end - self.position).min(self.context.batch_size as
u64);
+ let mut records = Vec::with_capacity(count as usize);
+ // Bound physical decoding too: sparse ranges must not turn a single
+ // non-cancellable blocking task into an entire-block record scan.
+ for _ in 0..count {
+ if self.selection.is_finished() {
+ break;
+ }
+ let record = self
+ .reader
+ .next()
+ .ok_or_else(|| avro_invalid("Truncated Avro block"))?
+ .map_err(avro_decode_error)?;
+ if self.selection.contains(self.position) {
+ records.push(record);
+ }
+ self.position += 1;
+ self.selection.advance_to(self.position);
+ }
+ let batch = if records.is_empty() {
+ None
+ } else {
+ let context = &self.context;
+ let batch = records_to_batch(&records, &context.fields,
&context.schema)?;
+ Some(match context.predicates.as_ref() {
+ Some(fp) =>
crate::arrow::residual::filter_record_batch_by_predicates(
+ batch,
+ fp,
+ &context.fields,
+ )?,
+ None => batch,
+ })
+ };
+ Ok((self, batch))
+ }
+}
+
+async fn avro_blocking<T: Send + 'static>(
+ task: impl FnOnce() -> crate::Result<T> + Send + 'static,
+) -> crate::Result<T> {
+ tokio::task::spawn_blocking(task)
Review Comment:
This introduces a panic for serial reads outside Tokio. I reproduced it with
a valid Avro file in memory-backed `FileIO`, polling the stream with
`futures::executor::block_on`. The first header decode panics here, before the
`JoinError` mapping can run. The previous reader worked on this executor. Could
we preserve that path with a fallback when `Handle::try_current()` finds no
runtime?
##########
crates/paimon/src/arrow/format/avro/streaming_tests/benchmark.rs:
##########
@@ -0,0 +1,185 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::*;
+use std::path::PathBuf;
+use std::sync::atomic::AtomicU64;
+use std::time::Instant;
+use tokio::io::{AsyncReadExt, AsyncSeekExt};
+
+const BLOCK_SIZE: usize = 1024 * 1024;
+const BATCH_SIZE: usize = 1024;
+const ROWS: usize = 2_000_000;
+const PAYLOAD_SIZE: usize = 128;
+
+struct BenchmarkRead {
+ file: tokio::sync::Mutex<tokio::fs::File>,
+ bytes_read: Arc<AtomicU64>,
+}
+
+#[async_trait]
+impl FileRead for BenchmarkRead {
+ async fn read(&self, range: Range<u64>) -> crate::Result<Bytes> {
+ let mut file = self.file.lock().await;
+ let mut buffer = vec![0; (range.end - range.start) as usize];
+ file.seek(std::io::SeekFrom::Start(range.start))
+ .await
+ .map_err(io_error)?;
+ file.read_exact(&mut buffer).await.map_err(io_error)?;
+ self.bytes_read
+ .fetch_add(buffer.len() as u64, Ordering::Relaxed);
+ Ok(buffer.into())
+ }
+}
+
+fn io_error(error: std::io::Error) -> Error {
+ Error::UnexpectedError {
+ message: "Avro benchmark I/O failed".into(),
+ source: Some(Box::new(error)),
+ }
+}
+
+async fn eager_stream(
+ reader: Box<dyn FileRead>,
+ size: u64,
+) -> crate::Result<ArrowRecordBatchStream> {
+ let bytes = reader.read(0..size).await?;
+ let records = Reader::new(bytes.as_ref())
+ .map_err(avro_decode_error)?
+ .collect::<std::result::Result<Vec<_>, _>>()
+ .map_err(avro_decode_error)?;
+ let fields = fields();
+ let schema = build_target_arrow_schema(&fields)?;
+ Ok(try_stream! {
+ for records in records.chunks(BATCH_SIZE) {
+ yield records_to_batch(records, &fields, &schema)?;
+ }
+ }
+ .boxed())
+}
+
+fn peak_rss_kib() -> Option<u64> {
+ std::fs::read_to_string("/proc/self/status")
+ .ok()?
+ .lines()
+ .find_map(|line| {
+ line.strip_prefix("VmHWM:")?
+ .split_whitespace()
+ .next()?
+ .parse()
+ .ok()
+ })
+}
+
+#[tokio::test]
+#[ignore = "manual Avro full-scan/LIMIT comparison; see
benchmarks/avro/README.md"]
+async fn compare_avro_reads() {
Review Comment:
Could you remove `streaming_tests/benchmark.rs` from this PR and put the
before/after full-scan and LIMIT measurements in the PR description? That would
let reviewers assess the trade-off without maintaining a second copy of the
eager reader in the test suite. Please also remove the module declaration and
update the README instructions that reference this file.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]