This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-11185-a9b8d27da30e52c778c10f2a76916a0af789fc00 in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
commit 081260215f5a1f4260d83213dc4598e5c63c950e Author: Phoenix <[email protected]> AuthorDate: Sat Oct 10 08:04:33 2026 +0000 test(parquet): move final writer roundtrips — Part IV (#11185) # Which issue does this PR close? Part of apache/arrow-rs#10540. # Stack base This draft PR targets `apache/arrow-rs:main`. Its review base is `consolidate-writer-roundtrip-III` (apache/arrow-rs#11184). Review the three commits added after that branch in the [stack delta](https://github.com/Phoenix500526/arrow-rs/compare/consolidate-writer-roundtrip-III...consolidate-writer-roundtrip-IV). # Rationale for this change Finish consolidating the remaining data roundtrip tests in the writer integration suite. # What changes are included in this PR? Move compatibility, encoding, and array roundtrip tests into `parquet/tests/arrow_writer/roundtrip.rs`. The array cases previously lived in writer and array-reader unit-test modules. Metadata, bloom-filter, error, and page-behavior tests remain with their focused unit suites. # Are these changes tested? - `cargo test --locked -p parquet --test arrow_writer` — 121 passed. - `cargo test --locked -p parquet --lib -- arrow::arrow_writer::tests` — 51 passed. - `cargo fmt --all -- --check` - `git diff --check consolidate-writer-roundtrip-III..HEAD` # Are there any user-facing changes? No. This is a test-only move. --- parquet/src/arrow/arrow_writer/mod.rs | 381 +------------------------------ parquet/tests/arrow_writer/roundtrip.rs | 391 +++++++++++++++++++++++++++++++- 2 files changed, 386 insertions(+), 386 deletions(-) diff --git a/parquet/src/arrow/arrow_writer/mod.rs b/parquet/src/arrow/arrow_writer/mod.rs index 0bfb1c4f5e..b8258b8de0 100644 --- a/parquet/src/arrow/arrow_writer/mod.rs +++ b/parquet/src/arrow/arrow_writer/mod.rs @@ -2091,8 +2091,8 @@ mod tests { use std::fs::File; + use crate::arrow::ARROW_SCHEMA_META_KEY; use crate::arrow::arrow_reader::{ParquetRecordBatchReader, ParquetRecordBatchReaderBuilder}; - use crate::arrow::{ARROW_SCHEMA_META_KEY, PARQUET_FIELD_ID_META_KEY}; use crate::column::page::{Page, PageReader}; use crate::file::metadata::thrift::PageHeader; use crate::file::page_index::column_index::ColumnIndexMetaData; @@ -2104,9 +2104,8 @@ mod tests { use arrow::util::data_gen::create_random_array; use arrow::util::pretty::pretty_format_batches; use arrow::{array::*, buffer::Buffer}; - use arrow_buffer::{IntervalMonthDayNano, NullBuffer, OffsetBuffer}; + use arrow_buffer::{IntervalMonthDayNano, NullBuffer}; use arrow_schema::Fields; - use tempfile::tempfile; use crate::basic::{Encoding, EncodingMask}; use crate::data_type::AsBytes; @@ -3004,237 +3003,6 @@ mod tests { } } - #[test] - fn arrow_writer_test_type_compatibility() { - fn ensure_compatible_write<T1, T2>(array1: T1, array2: T2, expected_result: T1) - where - T1: Array + 'static, - T2: Array + 'static, - { - let schema1 = Arc::new(Schema::new(vec![Field::new( - "a", - array1.data_type().clone(), - false, - )])); - - let file = tempfile().unwrap(); - let mut writer = - ArrowWriter::try_new(file.try_clone().unwrap(), schema1.clone(), None).unwrap(); - - let rb1 = RecordBatch::try_new(schema1.clone(), vec![Arc::new(array1)]).unwrap(); - writer.write(&rb1).unwrap(); - - let schema2 = Arc::new(Schema::new(vec![Field::new( - "a", - array2.data_type().clone(), - false, - )])); - let rb2 = RecordBatch::try_new(schema2, vec![Arc::new(array2)]).unwrap(); - writer.write(&rb2).unwrap(); - - writer.close().unwrap(); - - let mut record_batch_reader = - ParquetRecordBatchReader::try_new(file.try_clone().unwrap(), 1024).unwrap(); - let actual_batch = record_batch_reader.next().unwrap().unwrap(); - - let expected_batch = - RecordBatch::try_new(schema1, vec![Arc::new(expected_result)]).unwrap(); - assert_eq!(actual_batch, expected_batch); - } - - // check compatibility between native and dictionaries - - ensure_compatible_write( - DictionaryArray::new( - UInt8Array::from_iter_values(vec![0]), - Arc::new(StringArray::from_iter_values(vec!["parquet"])), - ), - StringArray::from_iter_values(vec!["barquet"]), - DictionaryArray::new( - UInt8Array::from_iter_values(vec![0, 1]), - Arc::new(StringArray::from_iter_values(vec!["parquet", "barquet"])), - ), - ); - - ensure_compatible_write( - StringArray::from_iter_values(vec!["parquet"]), - DictionaryArray::new( - UInt8Array::from_iter_values(vec![0]), - Arc::new(StringArray::from_iter_values(vec!["barquet"])), - ), - StringArray::from_iter_values(vec!["parquet", "barquet"]), - ); - - // check compatibility between dictionaries with different key types - - ensure_compatible_write( - DictionaryArray::new( - UInt8Array::from_iter_values(vec![0]), - Arc::new(StringArray::from_iter_values(vec!["parquet"])), - ), - DictionaryArray::new( - UInt16Array::from_iter_values(vec![0]), - Arc::new(StringArray::from_iter_values(vec!["barquet"])), - ), - DictionaryArray::new( - UInt8Array::from_iter_values(vec![0, 1]), - Arc::new(StringArray::from_iter_values(vec!["parquet", "barquet"])), - ), - ); - - // check compatibility between dictionaries with different value types - ensure_compatible_write( - DictionaryArray::new( - UInt8Array::from_iter_values(vec![0]), - Arc::new(StringArray::from_iter_values(vec!["parquet"])), - ), - DictionaryArray::new( - UInt8Array::from_iter_values(vec![0]), - Arc::new(LargeStringArray::from_iter_values(vec!["barquet"])), - ), - DictionaryArray::new( - UInt8Array::from_iter_values(vec![0, 1]), - Arc::new(StringArray::from_iter_values(vec!["parquet", "barquet"])), - ), - ); - - // check compatibility between a dictionary and a native array with a different type - ensure_compatible_write( - DictionaryArray::new( - UInt8Array::from_iter_values(vec![0]), - Arc::new(StringArray::from_iter_values(vec!["parquet"])), - ), - LargeStringArray::from_iter_values(vec!["barquet"]), - DictionaryArray::new( - UInt8Array::from_iter_values(vec![0, 1]), - Arc::new(StringArray::from_iter_values(vec!["parquet", "barquet"])), - ), - ); - - // check compatibility for string types - - ensure_compatible_write( - StringArray::from_iter_values(vec!["parquet"]), - LargeStringArray::from_iter_values(vec!["barquet"]), - StringArray::from_iter_values(vec!["parquet", "barquet"]), - ); - - ensure_compatible_write( - LargeStringArray::from_iter_values(vec!["parquet"]), - StringArray::from_iter_values(vec!["barquet"]), - LargeStringArray::from_iter_values(vec!["parquet", "barquet"]), - ); - - ensure_compatible_write( - StringArray::from_iter_values(vec!["parquet"]), - StringViewArray::from_iter_values(vec!["barquet"]), - StringArray::from_iter_values(vec!["parquet", "barquet"]), - ); - - ensure_compatible_write( - StringViewArray::from_iter_values(vec!["parquet"]), - StringArray::from_iter_values(vec!["barquet"]), - StringViewArray::from_iter_values(vec!["parquet", "barquet"]), - ); - - ensure_compatible_write( - LargeStringArray::from_iter_values(vec!["parquet"]), - StringViewArray::from_iter_values(vec!["barquet"]), - LargeStringArray::from_iter_values(vec!["parquet", "barquet"]), - ); - - ensure_compatible_write( - StringViewArray::from_iter_values(vec!["parquet"]), - LargeStringArray::from_iter_values(vec!["barquet"]), - StringViewArray::from_iter_values(vec!["parquet", "barquet"]), - ); - - // check compatibility for binary types - - ensure_compatible_write( - BinaryArray::from_iter_values(vec![b"parquet"]), - LargeBinaryArray::from_iter_values(vec![b"barquet"]), - BinaryArray::from_iter_values(vec![b"parquet", b"barquet"]), - ); - - ensure_compatible_write( - LargeBinaryArray::from_iter_values(vec![b"parquet"]), - BinaryArray::from_iter_values(vec![b"barquet"]), - LargeBinaryArray::from_iter_values(vec![b"parquet", b"barquet"]), - ); - - ensure_compatible_write( - BinaryArray::from_iter_values(vec![b"parquet"]), - BinaryViewArray::from_iter_values(vec![b"barquet"]), - BinaryArray::from_iter_values(vec![b"parquet", b"barquet"]), - ); - - ensure_compatible_write( - BinaryViewArray::from_iter_values(vec![b"parquet"]), - BinaryArray::from_iter_values(vec![b"barquet"]), - BinaryViewArray::from_iter_values(vec![b"parquet", b"barquet"]), - ); - - ensure_compatible_write( - BinaryViewArray::from_iter_values(vec![b"parquet"]), - LargeBinaryArray::from_iter_values(vec![b"barquet"]), - BinaryViewArray::from_iter_values(vec![b"parquet", b"barquet"]), - ); - - ensure_compatible_write( - LargeBinaryArray::from_iter_values(vec![b"parquet"]), - BinaryViewArray::from_iter_values(vec![b"barquet"]), - LargeBinaryArray::from_iter_values(vec![b"parquet", b"barquet"]), - ); - - // check compatibility for list types - - let list_field_metadata = HashMap::from_iter(vec![( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )]); - let list_field = Field::new_list_field(DataType::Int32, false); - - let values1 = Arc::new(Int32Array::from(vec![0, 1, 2, 3, 4])); - let offsets1 = OffsetBuffer::new(vec![0, 2, 5].into()); - - let values2 = Arc::new(Int32Array::from(vec![5, 6, 7, 8, 9])); - let offsets2 = OffsetBuffer::new(vec![0, 3, 5].into()); - - let values_expected = Arc::new(Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9])); - let offsets_expected = OffsetBuffer::new(vec![0, 2, 5, 8, 10].into()); - - ensure_compatible_write( - // when the initial schema has the metadata ... - ListArray::try_new( - Arc::new( - list_field - .clone() - .with_metadata(list_field_metadata.clone()), - ), - offsets1, - values1, - None, - ) - .unwrap(), - // ... and some intermediate schema doesn't have the metadata - ListArray::try_new(Arc::new(list_field.clone()), offsets2, values2, None).unwrap(), - // ... the write will still go through, and the resulting schema will inherit the initial metadata - ListArray::try_new( - Arc::new( - list_field - .clone() - .with_metadata(list_field_metadata.clone()), - ), - offsets_expected, - values_expected, - None, - ) - .unwrap(), - ); - } - #[test] #[cfg_attr(miri, ignore)] // Takes too long fn u32_min_max() { @@ -3349,97 +3117,6 @@ mod tests { } } - #[test] - #[cfg_attr(miri, ignore)] // Takes too long - fn test_list_of_struct_roundtrip() { - // define schema - let int_field = Field::new("a", DataType::Int32, true); - let int_field2 = Field::new("b", DataType::Int32, true); - - let int_builder = Int32Builder::with_capacity(10); - let int_builder2 = Int32Builder::with_capacity(10); - - let struct_builder = StructBuilder::new( - vec![int_field, int_field2], - vec![Box::new(int_builder), Box::new(int_builder2)], - ); - let mut list_builder = ListBuilder::new(struct_builder); - - // Construct the following array - // [{a: 1, b: 2}], [], null, [null, null], [{a: null, b: 3}], [{a: 2, b: null}] - - // [{a: 1, b: 2}] - let values = list_builder.values(); - values - .field_builder::<Int32Builder>(0) - .unwrap() - .append_value(1); - values - .field_builder::<Int32Builder>(1) - .unwrap() - .append_value(2); - values.append(true); - list_builder.append(true); - - // [] - list_builder.append(true); - - // null - list_builder.append(false); - - // [null, null] - let values = list_builder.values(); - values - .field_builder::<Int32Builder>(0) - .unwrap() - .append_null(); - values - .field_builder::<Int32Builder>(1) - .unwrap() - .append_null(); - values.append(false); - values - .field_builder::<Int32Builder>(0) - .unwrap() - .append_null(); - values - .field_builder::<Int32Builder>(1) - .unwrap() - .append_null(); - values.append(false); - list_builder.append(true); - - // [{a: null, b: 3}] - let values = list_builder.values(); - values - .field_builder::<Int32Builder>(0) - .unwrap() - .append_null(); - values - .field_builder::<Int32Builder>(1) - .unwrap() - .append_value(3); - values.append(true); - list_builder.append(true); - - // [{a: 2, b: null}] - let values = list_builder.values(); - values - .field_builder::<Int32Builder>(0) - .unwrap() - .append_value(2); - values - .field_builder::<Int32Builder>(1) - .unwrap() - .append_null(); - values.append(true); - list_builder.append(true); - - let array = Arc::new(list_builder.finish()); - - RoundTripTest::new(array).run(); - } - fn row_group_sizes(metadata: &ParquetMetaData) -> Vec<i64> { metadata.row_groups().iter().map(|x| x.num_rows()).collect() } @@ -3653,30 +3330,6 @@ mod tests { writer.close().unwrap(); } - #[test] - fn test_arrow_writer_nullable() { - let batch_schema = Schema::new(vec![Field::new("int32", DataType::Int32, false)]); - let file_schema = Schema::new(vec![Field::new("int32", DataType::Int32, true)]); - let file_schema = Arc::new(file_schema); - - let batch = RecordBatch::try_new( - Arc::new(batch_schema), - vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4])) as _], - ) - .unwrap(); - - let mut buf = Vec::with_capacity(1024); - let mut writer = ArrowWriter::try_new(&mut buf, file_schema.clone(), None).unwrap(); - writer.write(&batch).unwrap(); - writer.close().unwrap(); - - let mut read = ParquetRecordBatchReader::try_new(Bytes::from(buf), 1024).unwrap(); - let back = read.next().unwrap().unwrap(); - assert_eq!(back.schema(), file_schema); - assert_ne!(back.schema(), batch.schema()); - assert_eq!(back.column(0).as_ref(), batch.column(0).as_ref()); - } - #[test] fn in_progress_accounting() { // define schema @@ -3987,36 +3640,6 @@ mod tests { ); } - #[test] - // https://github.com/apache/arrow-rs/issues/6988 - fn test_roundtrip_empty_schema() { - // create empty record batch with empty schema - let empty_batch = RecordBatch::try_new_with_options( - Arc::new(Schema::empty()), - vec![], - &RecordBatchOptions::default().with_row_count(Some(0)), - ) - .unwrap(); - - // write to parquet - let mut parquet_bytes: Vec<u8> = Vec::new(); - let mut writer = - ArrowWriter::try_new(&mut parquet_bytes, empty_batch.schema(), None).unwrap(); - writer.write(&empty_batch).unwrap(); - writer.close().unwrap(); - - // read from parquet - let bytes = Bytes::from(parquet_bytes); - let reader = ParquetRecordBatchReaderBuilder::try_new(bytes).unwrap(); - assert_eq!(reader.schema(), &empty_batch.schema()); - let batches: Vec<_> = reader - .build() - .unwrap() - .collect::<ArrowResult<Vec<_>>>() - .unwrap(); - assert_eq!(batches.len(), 0); - } - #[test] fn test_page_stats_not_written_by_default() { let string_field = Field::new("a", DataType::Utf8, false); diff --git a/parquet/tests/arrow_writer/roundtrip.rs b/parquet/tests/arrow_writer/roundtrip.rs index 2e0da18580..9a122fabe9 100644 --- a/parquet/tests/arrow_writer/roundtrip.rs +++ b/parquet/tests/arrow_writer/roundtrip.rs @@ -25,9 +25,10 @@ use std::collections::HashMap; use std::sync::Arc; use arrow::datatypes::ToByteSlice; +use arrow::error::Result as ArrowResult; use arrow_array::builder::{ - FixedSizeBinaryBuilder, ListBuilder, PrimitiveDictionaryBuilder, PrimitiveRunBuilder, - StringViewBuilder, + FixedSizeBinaryBuilder, Int32Builder, ListBuilder, PrimitiveDictionaryBuilder, + PrimitiveRunBuilder, StringViewBuilder, StructBuilder, }; use arrow_array::cast::AsArray; use arrow_array::types::{ @@ -42,25 +43,26 @@ use arrow_array::{ DurationSecondArray, FixedSizeBinaryArray, Float16Array, Float32Array, Float64Array, Int8Array, Int16Array, Int32Array, Int32DictionaryArray, Int32RunArray, Int64Array, IntervalDayTimeArray, IntervalYearMonthArray, LargeBinaryArray, LargeListArray, LargeListViewArray, LargeStringArray, - ListArray, ListViewArray, NullArray, PrimitiveArray, RecordBatch, RecordBatchReader, RunArray, - StringArray, StringViewArray, StructArray, Time32MillisecondArray, Time32SecondArray, - Time64MicrosecondArray, Time64NanosecondArray, TimestampMicrosecondArray, + ListArray, ListViewArray, NullArray, PrimitiveArray, RecordBatch, RecordBatchOptions, + RecordBatchReader, RunArray, StringArray, StringViewArray, StructArray, Time32MillisecondArray, + Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray, UInt8Array, UInt8DictionaryArray, UInt16Array, UInt32Array, UInt64Array, }; -use arrow_buffer::{ArrowNativeType, Buffer, IntervalDayTime, NullBuffer, i256}; +use arrow_buffer::{ArrowNativeType, Buffer, IntervalDayTime, NullBuffer, OffsetBuffer, i256}; use arrow_data::{ArrayData, ArrayDataBuilder}; use arrow_schema::{DataType as ArrowDataType, Field, Fields, Schema, SchemaRef, TimeUnit}; use bytes::Bytes; use half::f16; use num_traits::{FromPrimitive, PrimInt, ToPrimitive}; -use parquet::arrow::ArrowWriter; use parquet::arrow::arrow_reader::{ ArrowReaderOptions, ParquetRecordBatchReader, ParquetRecordBatchReaderBuilder, }; +use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY}; use parquet::basic::Type as PhysicalType; use parquet::errors::Result; use parquet::file::properties::WriterProperties; +use tempfile::tempfile; #[test] #[cfg_attr(miri, ignore)] // Takes too long @@ -1622,6 +1624,381 @@ fn interval_day_time_single_column() { ]); } +#[test] +fn arrow_writer_test_type_compatibility() { + fn ensure_compatible_write<T1, T2>(array1: T1, array2: T2, expected_result: T1) + where + T1: Array + 'static, + T2: Array + 'static, + { + let schema1 = Arc::new(Schema::new(vec![Field::new( + "a", + array1.data_type().clone(), + false, + )])); + + let file = tempfile().unwrap(); + let mut writer = + ArrowWriter::try_new(file.try_clone().unwrap(), schema1.clone(), None).unwrap(); + + let rb1 = RecordBatch::try_new(schema1.clone(), vec![Arc::new(array1)]).unwrap(); + writer.write(&rb1).unwrap(); + + let schema2 = Arc::new(Schema::new(vec![Field::new( + "a", + array2.data_type().clone(), + false, + )])); + let rb2 = RecordBatch::try_new(schema2, vec![Arc::new(array2)]).unwrap(); + writer.write(&rb2).unwrap(); + + writer.close().unwrap(); + + let mut record_batch_reader = + ParquetRecordBatchReader::try_new(file.try_clone().unwrap(), 1024).unwrap(); + let actual_batch = record_batch_reader.next().unwrap().unwrap(); + + let expected_batch = + RecordBatch::try_new(schema1, vec![Arc::new(expected_result)]).unwrap(); + assert_eq!(actual_batch, expected_batch); + } + + // check compatibility between native and dictionaries + + ensure_compatible_write( + DictionaryArray::new( + UInt8Array::from_iter_values(vec![0]), + Arc::new(StringArray::from_iter_values(vec!["parquet"])), + ), + StringArray::from_iter_values(vec!["barquet"]), + DictionaryArray::new( + UInt8Array::from_iter_values(vec![0, 1]), + Arc::new(StringArray::from_iter_values(vec!["parquet", "barquet"])), + ), + ); + + ensure_compatible_write( + StringArray::from_iter_values(vec!["parquet"]), + DictionaryArray::new( + UInt8Array::from_iter_values(vec![0]), + Arc::new(StringArray::from_iter_values(vec!["barquet"])), + ), + StringArray::from_iter_values(vec!["parquet", "barquet"]), + ); + + // check compatibility between dictionaries with different key types + + ensure_compatible_write( + DictionaryArray::new( + UInt8Array::from_iter_values(vec![0]), + Arc::new(StringArray::from_iter_values(vec!["parquet"])), + ), + DictionaryArray::new( + UInt16Array::from_iter_values(vec![0]), + Arc::new(StringArray::from_iter_values(vec!["barquet"])), + ), + DictionaryArray::new( + UInt8Array::from_iter_values(vec![0, 1]), + Arc::new(StringArray::from_iter_values(vec!["parquet", "barquet"])), + ), + ); + + // check compatibility between dictionaries with different value types + ensure_compatible_write( + DictionaryArray::new( + UInt8Array::from_iter_values(vec![0]), + Arc::new(StringArray::from_iter_values(vec!["parquet"])), + ), + DictionaryArray::new( + UInt8Array::from_iter_values(vec![0]), + Arc::new(LargeStringArray::from_iter_values(vec!["barquet"])), + ), + DictionaryArray::new( + UInt8Array::from_iter_values(vec![0, 1]), + Arc::new(StringArray::from_iter_values(vec!["parquet", "barquet"])), + ), + ); + + // check compatibility between a dictionary and a native array with a different type + ensure_compatible_write( + DictionaryArray::new( + UInt8Array::from_iter_values(vec![0]), + Arc::new(StringArray::from_iter_values(vec!["parquet"])), + ), + LargeStringArray::from_iter_values(vec!["barquet"]), + DictionaryArray::new( + UInt8Array::from_iter_values(vec![0, 1]), + Arc::new(StringArray::from_iter_values(vec!["parquet", "barquet"])), + ), + ); + + // check compatibility for string types + + ensure_compatible_write( + StringArray::from_iter_values(vec!["parquet"]), + LargeStringArray::from_iter_values(vec!["barquet"]), + StringArray::from_iter_values(vec!["parquet", "barquet"]), + ); + + ensure_compatible_write( + LargeStringArray::from_iter_values(vec!["parquet"]), + StringArray::from_iter_values(vec!["barquet"]), + LargeStringArray::from_iter_values(vec!["parquet", "barquet"]), + ); + + ensure_compatible_write( + StringArray::from_iter_values(vec!["parquet"]), + StringViewArray::from_iter_values(vec!["barquet"]), + StringArray::from_iter_values(vec!["parquet", "barquet"]), + ); + + ensure_compatible_write( + StringViewArray::from_iter_values(vec!["parquet"]), + StringArray::from_iter_values(vec!["barquet"]), + StringViewArray::from_iter_values(vec!["parquet", "barquet"]), + ); + + ensure_compatible_write( + LargeStringArray::from_iter_values(vec!["parquet"]), + StringViewArray::from_iter_values(vec!["barquet"]), + LargeStringArray::from_iter_values(vec!["parquet", "barquet"]), + ); + + ensure_compatible_write( + StringViewArray::from_iter_values(vec!["parquet"]), + LargeStringArray::from_iter_values(vec!["barquet"]), + StringViewArray::from_iter_values(vec!["parquet", "barquet"]), + ); + + // check compatibility for binary types + + ensure_compatible_write( + BinaryArray::from_iter_values(vec![b"parquet"]), + LargeBinaryArray::from_iter_values(vec![b"barquet"]), + BinaryArray::from_iter_values(vec![b"parquet", b"barquet"]), + ); + + ensure_compatible_write( + LargeBinaryArray::from_iter_values(vec![b"parquet"]), + BinaryArray::from_iter_values(vec![b"barquet"]), + LargeBinaryArray::from_iter_values(vec![b"parquet", b"barquet"]), + ); + + ensure_compatible_write( + BinaryArray::from_iter_values(vec![b"parquet"]), + BinaryViewArray::from_iter_values(vec![b"barquet"]), + BinaryArray::from_iter_values(vec![b"parquet", b"barquet"]), + ); + + ensure_compatible_write( + BinaryViewArray::from_iter_values(vec![b"parquet"]), + BinaryArray::from_iter_values(vec![b"barquet"]), + BinaryViewArray::from_iter_values(vec![b"parquet", b"barquet"]), + ); + + ensure_compatible_write( + BinaryViewArray::from_iter_values(vec![b"parquet"]), + LargeBinaryArray::from_iter_values(vec![b"barquet"]), + BinaryViewArray::from_iter_values(vec![b"parquet", b"barquet"]), + ); + + ensure_compatible_write( + LargeBinaryArray::from_iter_values(vec![b"parquet"]), + BinaryViewArray::from_iter_values(vec![b"barquet"]), + LargeBinaryArray::from_iter_values(vec![b"parquet", b"barquet"]), + ); + + // check compatibility for list types + + let list_field_metadata = HashMap::from_iter(vec![( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )]); + let list_field = Field::new_list_field(ArrowDataType::Int32, false); + + let values1 = Arc::new(Int32Array::from(vec![0, 1, 2, 3, 4])); + let offsets1 = OffsetBuffer::new(vec![0, 2, 5].into()); + + let values2 = Arc::new(Int32Array::from(vec![5, 6, 7, 8, 9])); + let offsets2 = OffsetBuffer::new(vec![0, 3, 5].into()); + + let values_expected = Arc::new(Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9])); + let offsets_expected = OffsetBuffer::new(vec![0, 2, 5, 8, 10].into()); + + ensure_compatible_write( + // when the initial schema has the metadata ... + ListArray::try_new( + Arc::new( + list_field + .clone() + .with_metadata(list_field_metadata.clone()), + ), + offsets1, + values1, + None, + ) + .unwrap(), + // ... and some intermediate schema doesn't have the metadata + ListArray::try_new(Arc::new(list_field.clone()), offsets2, values2, None).unwrap(), + // ... the write will still go through, and the resulting schema will inherit the initial metadata + ListArray::try_new( + Arc::new( + list_field + .clone() + .with_metadata(list_field_metadata.clone()), + ), + offsets_expected, + values_expected, + None, + ) + .unwrap(), + ); +} + +#[test] +#[cfg_attr(miri, ignore)] // Takes too long +fn test_list_of_struct_roundtrip() { + // define schema + let int_field = Field::new("a", ArrowDataType::Int32, true); + let int_field2 = Field::new("b", ArrowDataType::Int32, true); + + let int_builder = Int32Builder::with_capacity(10); + let int_builder2 = Int32Builder::with_capacity(10); + + let struct_builder = StructBuilder::new( + vec![int_field, int_field2], + vec![Box::new(int_builder), Box::new(int_builder2)], + ); + let mut list_builder = ListBuilder::new(struct_builder); + + // Construct the following array + // [{a: 1, b: 2}], [], null, [null, null], [{a: null, b: 3}], [{a: 2, b: null}] + + // [{a: 1, b: 2}] + let values = list_builder.values(); + values + .field_builder::<Int32Builder>(0) + .unwrap() + .append_value(1); + values + .field_builder::<Int32Builder>(1) + .unwrap() + .append_value(2); + values.append(true); + list_builder.append(true); + + // [] + list_builder.append(true); + + // null + list_builder.append(false); + + // [null, null] + let values = list_builder.values(); + values + .field_builder::<Int32Builder>(0) + .unwrap() + .append_null(); + values + .field_builder::<Int32Builder>(1) + .unwrap() + .append_null(); + values.append(false); + values + .field_builder::<Int32Builder>(0) + .unwrap() + .append_null(); + values + .field_builder::<Int32Builder>(1) + .unwrap() + .append_null(); + values.append(false); + list_builder.append(true); + + // [{a: null, b: 3}] + let values = list_builder.values(); + values + .field_builder::<Int32Builder>(0) + .unwrap() + .append_null(); + values + .field_builder::<Int32Builder>(1) + .unwrap() + .append_value(3); + values.append(true); + list_builder.append(true); + + // [{a: 2, b: null}] + let values = list_builder.values(); + values + .field_builder::<Int32Builder>(0) + .unwrap() + .append_value(2); + values + .field_builder::<Int32Builder>(1) + .unwrap() + .append_null(); + values.append(true); + list_builder.append(true); + + let array = Arc::new(list_builder.finish()); + + RoundTripTest::new(array).run(); +} + +#[test] +// https://github.com/apache/arrow-rs/issues/6988 +fn test_roundtrip_empty_schema() { + // create empty record batch with empty schema + let empty_batch = RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + vec![], + &RecordBatchOptions::default().with_row_count(Some(0)), + ) + .unwrap(); + + // write to parquet + let mut parquet_bytes: Vec<u8> = Vec::new(); + let mut writer = ArrowWriter::try_new(&mut parquet_bytes, empty_batch.schema(), None).unwrap(); + writer.write(&empty_batch).unwrap(); + writer.close().unwrap(); + + // read from parquet + let bytes = Bytes::from(parquet_bytes); + let reader = ParquetRecordBatchReaderBuilder::try_new(bytes).unwrap(); + assert_eq!(reader.schema(), &empty_batch.schema()); + let batches: Vec<_> = reader + .build() + .unwrap() + .collect::<ArrowResult<Vec<_>>>() + .unwrap(); + assert_eq!(batches.len(), 0); +} + +#[test] +fn test_arrow_writer_nullable() { + let batch_schema = Schema::new(vec![Field::new("int32", ArrowDataType::Int32, false)]); + let file_schema = Schema::new(vec![Field::new("int32", ArrowDataType::Int32, true)]); + let file_schema = Arc::new(file_schema); + + let batch = RecordBatch::try_new( + Arc::new(batch_schema), + vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4])) as _], + ) + .unwrap(); + + let mut buf = Vec::with_capacity(1024); + let mut writer = ArrowWriter::try_new(&mut buf, file_schema.clone(), None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let mut read = ParquetRecordBatchReader::try_new(Bytes::from(buf), 1024).unwrap(); + let back = read.next().unwrap().unwrap(); + assert_eq!(back.schema(), file_schema); + assert_ne!(back.schema(), batch.schema()); + assert_eq!(back.column(0).as_ref(), batch.column(0).as_ref()); +} + #[test] fn test_unsigned_roundtrip() { let schema = Arc::new(Schema::new(vec