alamb commented on code in PR #10128:
URL: https://github.com/apache/arrow-rs/pull/10128#discussion_r4234511530
##########
arrow-ipc/src/writer.rs:
##########
@@ -604,27 +635,25 @@ impl IpcDataGenerator {
}
}
- fn _encode_dictionaries<I: Iterator<Item = i64>>(
+ fn _collect_dict_updates<I: Iterator<Item = i64>>(
&self,
column: &ArrayRef,
- encoded_dictionaries: &mut Vec<EncodedData>,
+ encoded_dictionaries: &mut Vec<DictionaryToEncode>,
Review Comment:
the parameter is now somewhat confusingly named as it is't actually encoded
dictionaries. Maybe renaming it to `dictionaries_to_encode` would make this
clearer
##########
arrow-ipc/src/writer.rs:
##########
@@ -318,23 +324,27 @@ where
{
fn write_record_batch(
&mut self,
- metadata: Vec<u8>,
- encoded_buffers: Vec<EncodedBuffer>,
+ metadata: &[u8],
+ encoded_buffers: &mut Vec<EncodedBuffer>,
body_len: usize,
- tail_pad: usize,
write_options: &IpcWriteOptions,
) -> Result<(usize, usize), ArrowError> {
let alignment = write_options.alignment;
let layout = MetadataLayout::new(metadata.len(), write_options);
self.write_continuation(write_options, layout.padded_metadata_len as
i32)?;
- self.write_all(&metadata)?;
+ self.write_all(metadata)?;
self.write_all(&PADDING[..layout.metadata_padding])?;
- for enc in &encoded_buffers {
+ for enc in encoded_buffers.iter_mut() {
self.write_all(enc.as_slice())?;
self.write_all(&PADDING[..pad_to_alignment(alignment,
enc.len())])?;
}
- self.write_all(&PADDING[..tail_pad])?;
+ // Clearing the buffers after the loop instead of draining the vec is
Review Comment:
I don't understand this -- `encoded_buffers` is Vec<EncodedBuffer>` but each
EncodedBuffer is just a few pointers I think. I see
https://github.com/apache/arrow-rs/pull/10128#issuecomment-5406019272 but am
still confused
I wonder if that is some sort of benchmarking effect
##########
arrow-ipc/src/writer.rs:
##########
@@ -763,16 +783,17 @@ impl IpcDataGenerator {
Ok(())
}
- #[expect(clippy::too_many_arguments)]
- fn encode_dictionaries<I: Iterator<Item = i64>>(
+ // Collect all dicts that need a dictionary message i.e. ones that were
+ // either not in the tracker previously or ones that were but need a
+ // replacement or delta.
+ fn collect_dict_updates<I: Iterator<Item = i64>>(
&self,
field: &Field,
column: &ArrayRef,
- encoded_dictionaries: &mut Vec<EncodedData>,
+ dictionaries: &mut Vec<DictionaryToEncode>,
Review Comment:
it would be nice if this parameter was named consistently with
`_collect_dict_updates` -- perhaps `dictionaries_to_encode`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
