Rich-T-kid commented on code in PR #10044:
URL: https://github.com/apache/arrow-rs/pull/10044#discussion_r3381017151
##########
arrow-ipc/src/writer.rs:
##########
@@ -474,27 +523,116 @@ impl IpcDataGenerator {
write_options: &IpcWriteOptions,
compression_context: &mut CompressionContext,
) -> Result<(Vec<EncodedData>, EncodedData), ArrowError> {
+ let encoded_dictionaries = self.encode_all_dicts(
+ batch,
+ dictionary_tracker,
+ write_options,
+ compression_context,
+ )?;
+ let mut arrow_data = Vec::new();
+ let (ipc_message, _, tail_pad) = self.record_batch_to_bytes(
+ batch,
+ write_options,
+ compression_context,
+ &mut BufferSink::Write(&mut arrow_data),
+ )?;
+ arrow_data.extend_from_slice(&PADDING[..tail_pad]);
+ Ok((
+ encoded_dictionaries,
+ EncodedData {
+ ipc_message,
+ arrow_data,
+ },
+ ))
+ }
+
+ /// Encode dictionary batches for all columns in `batch`.
+ fn encode_all_dicts(
+ &self,
+ batch: &RecordBatch,
+ dictionary_tracker: &mut DictionaryTracker,
+ write_options: &IpcWriteOptions,
+ compression_context: &mut CompressionContext,
+ ) -> Result<Vec<EncodedData>, ArrowError> {
let schema = batch.schema();
let mut encoded_dictionaries =
Vec::with_capacity(schema.flattened_fields().len());
-
let mut dict_id = dictionary_tracker.dict_ids.clone().into_iter();
-
for (i, field) in schema.fields().iter().enumerate() {
- let column = batch.column(i);
self.encode_dictionaries(
field,
- column,
+ batch.column(i),
&mut encoded_dictionaries,
dictionary_tracker,
write_options,
&mut dict_id,
compression_context,
)?;
}
+ Ok(encoded_dictionaries)
+ }
- let encoded_message =
- self.record_batch_to_bytes(batch, write_options,
compression_context)?;
- Ok((encoded_dictionaries, encoded_message))
+ /// Write dictionary batches and the record batch directly to `writer`,
skipping the
+ /// intermediate body `Vec<u8>` allocations
+ /// Returns [`IpcWriteMetadata`] with the sizes needed to build footer
blocks.
+ fn write_direct<W: Write>(
Review Comment:
nope, make sense to me.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
