This is an automated email from the ASF dual-hosted git repository.
Jefffrey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new d3d7ae83e7 fix(ipc): reject dictionary-encoded dictionary values
(#10230)
d3d7ae83e7 is described below
commit d3d7ae83e7ca2678be8e4ad950d5c4a087ff8efb
Author: Goutam Adwant <[email protected]>
AuthorDate: Thu Jul 2 01:20:17 2026 -0700
fix(ipc): reject dictionary-encoded dictionary values (#10230)
# Which issue does this PR close?
- Closes #10213.
# Rationale for this change
Arrow IPC dictionary batches write a dictionary field's values as the
batch payload. When those values are themselves dictionary-encoded, the
writer can produce IPC data that readers cannot decode, failing later
with a buffer metadata mismatch.
# What changes are included in this PR?
This adds schema validation to the IPC file and stream writer
constructors so direct dictionary-of-dictionary schemas return a clear
`InvalidArgumentError` before any IPC bytes are written. A low-level
dictionary encoding guard is kept as a backstop.
A regression test covers the direct `Dictionary(_, Dictionary(_, _))`
case for both IPC stream and file writers.
# Are these changes tested?
Yes.
- `cargo fmt --all -- --check`
- `cargo test -p arrow-ipc`
- `cargo test -p arrow-ipc --all-features`
- `cargo clippy -p arrow-ipc --all-targets -- -D warnings`
- `cargo clippy -p arrow-ipc --all-targets --all-features -- -D
warnings`
# Are there any user-facing changes?
Yes. The IPC file and stream writers now return a clear error for direct
dictionary-of-dictionary schemas instead of writing IPC data that fails
during read. There are no public API changes.
---
arrow-ipc/src/reader.rs | 42 ++++++++++++++++++++++++++++++++++++++
arrow-ipc/src/writer.rs | 54 ++++++++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 95 insertions(+), 1 deletion(-)
diff --git a/arrow-ipc/src/reader.rs b/arrow-ipc/src/reader.rs
index e6f45c4cd0..6cb1d7e6b0 100644
--- a/arrow-ipc/src/reader.rs
+++ b/arrow-ipc/src/reader.rs
@@ -2693,6 +2693,48 @@ mod tests {
assert_eq!(input_batch, output_batch);
}
+ #[test]
+ fn test_ipc_writers_reject_dictionary_of_dictionary_schema() {
+ let values = Arc::new(StringArray::from(vec![Some("a"), Some("b")]))
as ArrayRef;
+ let inner = Arc::new(DictionaryArray::new(
+ UInt32Array::from_iter_values([0, 1]),
+ values,
+ )) as ArrayRef;
+ let outer = Arc::new(DictionaryArray::new(
+ UInt32Array::from_iter_values([0, 1, 0]),
+ inner,
+ )) as ArrayRef;
+
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "f1",
+ outer.data_type().clone(),
+ false,
+ )]));
+ let batch = RecordBatch::try_new(schema, vec![outer]).unwrap();
+
+ let mut stream = Vec::new();
+ let Err(err) = crate::writer::StreamWriter::try_new(&mut stream,
batch.schema_ref()) else {
+ panic!("IPC stream writer should reject dictionary-of-dictionary
schemas");
+ };
+ assert!(stream.is_empty());
+
+ assert!(
+ err.to_string().contains("dictionary-of-dictionary values"),
+ "unexpected error: {err}"
+ );
+
+ let mut file = Vec::new();
+ let Err(err) = crate::writer::FileWriter::try_new(&mut file,
batch.schema_ref()) else {
+ panic!("IPC file writer should reject dictionary-of-dictionary
schemas");
+ };
+ assert!(file.is_empty());
+
+ assert!(
+ err.to_string().contains("dictionary-of-dictionary values"),
+ "unexpected error: {err}"
+ );
+ }
+
#[test]
fn test_roundtrip_stream_nested_dict_of_map_of_dict() {
let values = StringArray::from(vec![Some("a"), None, Some("b"),
Some("c")]);
diff --git a/arrow-ipc/src/writer.rs b/arrow-ipc/src/writer.rs
index 618f8b2fc9..6ae6484373 100644
--- a/arrow-ipc/src/writer.rs
+++ b/arrow-ipc/src/writer.rs
@@ -537,7 +537,14 @@ impl IpcDataGenerator {
ipc_write_context: &mut IpcWriteContext,
) -> Result<(), ArrowError> {
match column.data_type() {
- DataType::Dictionary(_key_type, _value_type) => {
+ DataType::Dictionary(_key_type, value_type) => {
+ if matches!(value_type.as_ref(), DataType::Dictionary(_, _)) {
+ return Err(ArrowError::InvalidArgumentError(format!(
+ "Arrow IPC field metadata cannot encode direct
dictionary-of-dictionary values for field {:?}",
+ field.name()
+ )));
+ }
+
let dict_data = column.to_data();
let dict_values = &dict_data.child_data()[0];
@@ -929,6 +936,47 @@ impl IpcDataGenerator {
}
}
+fn ensure_supported_ipc_schema(schema: &Schema) -> Result<(), ArrowError> {
+ schema
+ .fields()
+ .iter()
+ .try_for_each(|field| ensure_supported_ipc_data_type(field.name(),
field.data_type()))
+}
+
+fn ensure_supported_ipc_data_type(
+ field_name: &str,
+ data_type: &DataType,
+) -> Result<(), ArrowError> {
+ match data_type {
+ DataType::Dictionary(_, value_type)
+ if matches!(value_type.as_ref(), DataType::Dictionary(_, _)) =>
+ {
+ Err(ArrowError::InvalidArgumentError(format!(
+ "Arrow IPC field metadata cannot encode direct
dictionary-of-dictionary values for field {field_name:?}"
+ )))
+ }
+ DataType::Dictionary(_, value_type) => {
+ ensure_supported_ipc_data_type(field_name, value_type)
+ }
+ DataType::Struct(fields) => fields
+ .iter()
+ .try_for_each(|field| ensure_supported_ipc_data_type(field.name(),
field.data_type())),
+ DataType::RunEndEncoded(_, field)
+ | DataType::List(field)
+ | DataType::LargeList(field)
+ | DataType::ListView(field)
+ | DataType::LargeListView(field)
+ | DataType::FixedSizeList(field, _)
+ | DataType::Map(field, _) => {
+ ensure_supported_ipc_data_type(field.name(), field.data_type())
+ }
+ DataType::Union(fields, _) => fields.iter().try_for_each(|(_, field)| {
+ ensure_supported_ipc_data_type(field.name(), field.data_type())
+ }),
+ _ => Ok(()),
+ }
+}
+
fn append_variadic_buffer_counts(counts: &mut Vec<i64>, array: &ArrayData) {
match array.data_type() {
DataType::BinaryView | DataType::Utf8View => {
@@ -1344,6 +1392,8 @@ impl<W: Write> FileWriter<W> {
schema: &Schema,
write_options: IpcWriteOptions,
) -> Result<Self, ArrowError> {
+ ensure_supported_ipc_schema(schema)?;
+
let data_gen = IpcDataGenerator::default();
// write magic to header aligned on alignment boundary
let pad_len = pad_to_alignment(write_options.alignment,
super::ARROW_MAGIC.len());
@@ -1633,6 +1683,8 @@ impl<W: Write> StreamWriter<W> {
schema: &Schema,
write_options: IpcWriteOptions,
) -> Result<Self, ArrowError> {
+ ensure_supported_ipc_schema(schema)?;
+
let data_gen = IpcDataGenerator::default();
let mut dictionary_tracker = DictionaryTracker::new(false);
