This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 081260215f test(parquet): move final writer roundtrips — Part IV
(#11185)
081260215f is described below
commit 081260215f5a1f4260d83213dc4598e5c63c950e
Author: Phoenix <[email protected]>
AuthorDate: Sat Oct 10 08:04:33 2026 +0000
test(parquet): move final writer roundtrips — Part IV (#11185)
# Which issue does this PR close?
Part of apache/arrow-rs#10540.
# Stack base
This draft PR targets `apache/arrow-rs:main`. Its review base is
`consolidate-writer-roundtrip-III` (apache/arrow-rs#11184). Review the
three commits added after that branch in the [stack
delta](https://github.com/Phoenix500526/arrow-rs/compare/consolidate-writer-roundtrip-III...consolidate-writer-roundtrip-IV).
# Rationale for this change
Finish consolidating the remaining data roundtrip tests in the writer
integration suite.
# What changes are included in this PR?
Move compatibility, encoding, and array roundtrip tests into
`parquet/tests/arrow_writer/roundtrip.rs`. The array cases previously
lived in writer and array-reader unit-test modules. Metadata,
bloom-filter, error, and page-behavior tests remain with their focused
unit suites.
# Are these changes tested?
- `cargo test --locked -p parquet --test arrow_writer` — 121 passed.
- `cargo test --locked -p parquet --lib -- arrow::arrow_writer::tests` —
51 passed.
- `cargo fmt --all -- --check`
- `git diff --check consolidate-writer-roundtrip-III..HEAD`
# Are there any user-facing changes?
No. This is a test-only move.
---
parquet/src/arrow/arrow_writer/mod.rs | 381 +------------------------------
parquet/tests/arrow_writer/roundtrip.rs | 391 +++++++++++++++++++++++++++++++-
2 files changed, 386 insertions(+), 386 deletions(-)
diff --git a/parquet/src/arrow/arrow_writer/mod.rs
b/parquet/src/arrow/arrow_writer/mod.rs
index 0bfb1c4f5e..b8258b8de0 100644
--- a/parquet/src/arrow/arrow_writer/mod.rs
+++ b/parquet/src/arrow/arrow_writer/mod.rs
@@ -2091,8 +2091,8 @@ mod tests {
use std::fs::File;
+ use crate::arrow::ARROW_SCHEMA_META_KEY;
use crate::arrow::arrow_reader::{ParquetRecordBatchReader,
ParquetRecordBatchReaderBuilder};
- use crate::arrow::{ARROW_SCHEMA_META_KEY, PARQUET_FIELD_ID_META_KEY};
use crate::column::page::{Page, PageReader};
use crate::file::metadata::thrift::PageHeader;
use crate::file::page_index::column_index::ColumnIndexMetaData;
@@ -2104,9 +2104,8 @@ mod tests {
use arrow::util::data_gen::create_random_array;
use arrow::util::pretty::pretty_format_batches;
use arrow::{array::*, buffer::Buffer};
- use arrow_buffer::{IntervalMonthDayNano, NullBuffer, OffsetBuffer};
+ use arrow_buffer::{IntervalMonthDayNano, NullBuffer};
use arrow_schema::Fields;
- use tempfile::tempfile;
use crate::basic::{Encoding, EncodingMask};
use crate::data_type::AsBytes;
@@ -3004,237 +3003,6 @@ mod tests {
}
}
- #[test]
- fn arrow_writer_test_type_compatibility() {
- fn ensure_compatible_write<T1, T2>(array1: T1, array2: T2,
expected_result: T1)
- where
- T1: Array + 'static,
- T2: Array + 'static,
- {
- let schema1 = Arc::new(Schema::new(vec![Field::new(
- "a",
- array1.data_type().clone(),
- false,
- )]));
-
- let file = tempfile().unwrap();
- let mut writer =
- ArrowWriter::try_new(file.try_clone().unwrap(),
schema1.clone(), None).unwrap();
-
- let rb1 = RecordBatch::try_new(schema1.clone(),
vec![Arc::new(array1)]).unwrap();
- writer.write(&rb1).unwrap();
-
- let schema2 = Arc::new(Schema::new(vec![Field::new(
- "a",
- array2.data_type().clone(),
- false,
- )]));
- let rb2 = RecordBatch::try_new(schema2,
vec![Arc::new(array2)]).unwrap();
- writer.write(&rb2).unwrap();
-
- writer.close().unwrap();
-
- let mut record_batch_reader =
- ParquetRecordBatchReader::try_new(file.try_clone().unwrap(),
1024).unwrap();
- let actual_batch = record_batch_reader.next().unwrap().unwrap();
-
- let expected_batch =
- RecordBatch::try_new(schema1,
vec![Arc::new(expected_result)]).unwrap();
- assert_eq!(actual_batch, expected_batch);
- }
-
- // check compatibility between native and dictionaries
-
- ensure_compatible_write(
- DictionaryArray::new(
- UInt8Array::from_iter_values(vec![0]),
- Arc::new(StringArray::from_iter_values(vec!["parquet"])),
- ),
- StringArray::from_iter_values(vec!["barquet"]),
- DictionaryArray::new(
- UInt8Array::from_iter_values(vec![0, 1]),
- Arc::new(StringArray::from_iter_values(vec!["parquet",
"barquet"])),
- ),
- );
-
- ensure_compatible_write(
- StringArray::from_iter_values(vec!["parquet"]),
- DictionaryArray::new(
- UInt8Array::from_iter_values(vec![0]),
- Arc::new(StringArray::from_iter_values(vec!["barquet"])),
- ),
- StringArray::from_iter_values(vec!["parquet", "barquet"]),
- );
-
- // check compatibility between dictionaries with different key types
-
- ensure_compatible_write(
- DictionaryArray::new(
- UInt8Array::from_iter_values(vec![0]),
- Arc::new(StringArray::from_iter_values(vec!["parquet"])),
- ),
- DictionaryArray::new(
- UInt16Array::from_iter_values(vec![0]),
- Arc::new(StringArray::from_iter_values(vec!["barquet"])),
- ),
- DictionaryArray::new(
- UInt8Array::from_iter_values(vec![0, 1]),
- Arc::new(StringArray::from_iter_values(vec!["parquet",
"barquet"])),
- ),
- );
-
- // check compatibility between dictionaries with different value types
- ensure_compatible_write(
- DictionaryArray::new(
- UInt8Array::from_iter_values(vec![0]),
- Arc::new(StringArray::from_iter_values(vec!["parquet"])),
- ),
- DictionaryArray::new(
- UInt8Array::from_iter_values(vec![0]),
- Arc::new(LargeStringArray::from_iter_values(vec!["barquet"])),
- ),
- DictionaryArray::new(
- UInt8Array::from_iter_values(vec![0, 1]),
- Arc::new(StringArray::from_iter_values(vec!["parquet",
"barquet"])),
- ),
- );
-
- // check compatibility between a dictionary and a native array with a
different type
- ensure_compatible_write(
- DictionaryArray::new(
- UInt8Array::from_iter_values(vec![0]),
- Arc::new(StringArray::from_iter_values(vec!["parquet"])),
- ),
- LargeStringArray::from_iter_values(vec!["barquet"]),
- DictionaryArray::new(
- UInt8Array::from_iter_values(vec![0, 1]),
- Arc::new(StringArray::from_iter_values(vec!["parquet",
"barquet"])),
- ),
- );
-
- // check compatibility for string types
-
- ensure_compatible_write(
- StringArray::from_iter_values(vec!["parquet"]),
- LargeStringArray::from_iter_values(vec!["barquet"]),
- StringArray::from_iter_values(vec!["parquet", "barquet"]),
- );
-
- ensure_compatible_write(
- LargeStringArray::from_iter_values(vec!["parquet"]),
- StringArray::from_iter_values(vec!["barquet"]),
- LargeStringArray::from_iter_values(vec!["parquet", "barquet"]),
- );
-
- ensure_compatible_write(
- StringArray::from_iter_values(vec!["parquet"]),
- StringViewArray::from_iter_values(vec!["barquet"]),
- StringArray::from_iter_values(vec!["parquet", "barquet"]),
- );
-
- ensure_compatible_write(
- StringViewArray::from_iter_values(vec!["parquet"]),
- StringArray::from_iter_values(vec!["barquet"]),
- StringViewArray::from_iter_values(vec!["parquet", "barquet"]),
- );
-
- ensure_compatible_write(
- LargeStringArray::from_iter_values(vec!["parquet"]),
- StringViewArray::from_iter_values(vec!["barquet"]),
- LargeStringArray::from_iter_values(vec!["parquet", "barquet"]),
- );
-
- ensure_compatible_write(
- StringViewArray::from_iter_values(vec!["parquet"]),
- LargeStringArray::from_iter_values(vec!["barquet"]),
- StringViewArray::from_iter_values(vec!["parquet", "barquet"]),
- );
-
- // check compatibility for binary types
-
- ensure_compatible_write(
- BinaryArray::from_iter_values(vec![b"parquet"]),
- LargeBinaryArray::from_iter_values(vec![b"barquet"]),
- BinaryArray::from_iter_values(vec![b"parquet", b"barquet"]),
- );
-
- ensure_compatible_write(
- LargeBinaryArray::from_iter_values(vec![b"parquet"]),
- BinaryArray::from_iter_values(vec![b"barquet"]),
- LargeBinaryArray::from_iter_values(vec![b"parquet", b"barquet"]),
- );
-
- ensure_compatible_write(
- BinaryArray::from_iter_values(vec![b"parquet"]),
- BinaryViewArray::from_iter_values(vec![b"barquet"]),
- BinaryArray::from_iter_values(vec![b"parquet", b"barquet"]),
- );
-
- ensure_compatible_write(
- BinaryViewArray::from_iter_values(vec![b"parquet"]),
- BinaryArray::from_iter_values(vec![b"barquet"]),
- BinaryViewArray::from_iter_values(vec![b"parquet", b"barquet"]),
- );
-
- ensure_compatible_write(
- BinaryViewArray::from_iter_values(vec![b"parquet"]),
- LargeBinaryArray::from_iter_values(vec![b"barquet"]),
- BinaryViewArray::from_iter_values(vec![b"parquet", b"barquet"]),
- );
-
- ensure_compatible_write(
- LargeBinaryArray::from_iter_values(vec![b"parquet"]),
- BinaryViewArray::from_iter_values(vec![b"barquet"]),
- LargeBinaryArray::from_iter_values(vec![b"parquet", b"barquet"]),
- );
-
- // check compatibility for list types
-
- let list_field_metadata = HashMap::from_iter(vec![(
- PARQUET_FIELD_ID_META_KEY.to_string(),
- "1".to_string(),
- )]);
- let list_field = Field::new_list_field(DataType::Int32, false);
-
- let values1 = Arc::new(Int32Array::from(vec![0, 1, 2, 3, 4]));
- let offsets1 = OffsetBuffer::new(vec![0, 2, 5].into());
-
- let values2 = Arc::new(Int32Array::from(vec![5, 6, 7, 8, 9]));
- let offsets2 = OffsetBuffer::new(vec![0, 3, 5].into());
-
- let values_expected = Arc::new(Int32Array::from(vec![0, 1, 2, 3, 4, 5,
6, 7, 8, 9]));
- let offsets_expected = OffsetBuffer::new(vec![0, 2, 5, 8, 10].into());
-
- ensure_compatible_write(
- // when the initial schema has the metadata ...
- ListArray::try_new(
- Arc::new(
- list_field
- .clone()
- .with_metadata(list_field_metadata.clone()),
- ),
- offsets1,
- values1,
- None,
- )
- .unwrap(),
- // ... and some intermediate schema doesn't have the metadata
- ListArray::try_new(Arc::new(list_field.clone()), offsets2,
values2, None).unwrap(),
- // ... the write will still go through, and the resulting schema
will inherit the initial metadata
- ListArray::try_new(
- Arc::new(
- list_field
- .clone()
- .with_metadata(list_field_metadata.clone()),
- ),
- offsets_expected,
- values_expected,
- None,
- )
- .unwrap(),
- );
- }
-
#[test]
#[cfg_attr(miri, ignore)] // Takes too long
fn u32_min_max() {
@@ -3349,97 +3117,6 @@ mod tests {
}
}
- #[test]
- #[cfg_attr(miri, ignore)] // Takes too long
- fn test_list_of_struct_roundtrip() {
- // define schema
- let int_field = Field::new("a", DataType::Int32, true);
- let int_field2 = Field::new("b", DataType::Int32, true);
-
- let int_builder = Int32Builder::with_capacity(10);
- let int_builder2 = Int32Builder::with_capacity(10);
-
- let struct_builder = StructBuilder::new(
- vec![int_field, int_field2],
- vec![Box::new(int_builder), Box::new(int_builder2)],
- );
- let mut list_builder = ListBuilder::new(struct_builder);
-
- // Construct the following array
- // [{a: 1, b: 2}], [], null, [null, null], [{a: null, b: 3}], [{a: 2,
b: null}]
-
- // [{a: 1, b: 2}]
- let values = list_builder.values();
- values
- .field_builder::<Int32Builder>(0)
- .unwrap()
- .append_value(1);
- values
- .field_builder::<Int32Builder>(1)
- .unwrap()
- .append_value(2);
- values.append(true);
- list_builder.append(true);
-
- // []
- list_builder.append(true);
-
- // null
- list_builder.append(false);
-
- // [null, null]
- let values = list_builder.values();
- values
- .field_builder::<Int32Builder>(0)
- .unwrap()
- .append_null();
- values
- .field_builder::<Int32Builder>(1)
- .unwrap()
- .append_null();
- values.append(false);
- values
- .field_builder::<Int32Builder>(0)
- .unwrap()
- .append_null();
- values
- .field_builder::<Int32Builder>(1)
- .unwrap()
- .append_null();
- values.append(false);
- list_builder.append(true);
-
- // [{a: null, b: 3}]
- let values = list_builder.values();
- values
- .field_builder::<Int32Builder>(0)
- .unwrap()
- .append_null();
- values
- .field_builder::<Int32Builder>(1)
- .unwrap()
- .append_value(3);
- values.append(true);
- list_builder.append(true);
-
- // [{a: 2, b: null}]
- let values = list_builder.values();
- values
- .field_builder::<Int32Builder>(0)
- .unwrap()
- .append_value(2);
- values
- .field_builder::<Int32Builder>(1)
- .unwrap()
- .append_null();
- values.append(true);
- list_builder.append(true);
-
- let array = Arc::new(list_builder.finish());
-
- RoundTripTest::new(array).run();
- }
-
fn row_group_sizes(metadata: &ParquetMetaData) -> Vec<i64> {
metadata.row_groups().iter().map(|x| x.num_rows()).collect()
}
@@ -3653,30 +3330,6 @@ mod tests {
writer.close().unwrap();
}
- #[test]
- fn test_arrow_writer_nullable() {
- let batch_schema = Schema::new(vec![Field::new("int32",
DataType::Int32, false)]);
- let file_schema = Schema::new(vec![Field::new("int32",
DataType::Int32, true)]);
- let file_schema = Arc::new(file_schema);
-
- let batch = RecordBatch::try_new(
- Arc::new(batch_schema),
- vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4])) as _],
- )
- .unwrap();
-
- let mut buf = Vec::with_capacity(1024);
- let mut writer = ArrowWriter::try_new(&mut buf, file_schema.clone(),
None).unwrap();
- writer.write(&batch).unwrap();
- writer.close().unwrap();
-
- let mut read = ParquetRecordBatchReader::try_new(Bytes::from(buf),
1024).unwrap();
- let back = read.next().unwrap().unwrap();
- assert_eq!(back.schema(), file_schema);
- assert_ne!(back.schema(), batch.schema());
- assert_eq!(back.column(0).as_ref(), batch.column(0).as_ref());
- }
-
#[test]
fn in_progress_accounting() {
// define schema
@@ -3987,36 +3640,6 @@ mod tests {
);
}
- #[test]
- // https://github.com/apache/arrow-rs/issues/6988
- fn test_roundtrip_empty_schema() {
- // create empty record batch with empty schema
- let empty_batch = RecordBatch::try_new_with_options(
- Arc::new(Schema::empty()),
- vec![],
- &RecordBatchOptions::default().with_row_count(Some(0)),
- )
- .unwrap();
-
- // write to parquet
- let mut parquet_bytes: Vec<u8> = Vec::new();
- let mut writer =
- ArrowWriter::try_new(&mut parquet_bytes, empty_batch.schema(),
None).unwrap();
- writer.write(&empty_batch).unwrap();
- writer.close().unwrap();
-
- // read from parquet
- let bytes = Bytes::from(parquet_bytes);
- let reader = ParquetRecordBatchReaderBuilder::try_new(bytes).unwrap();
- assert_eq!(reader.schema(), &empty_batch.schema());
- let batches: Vec<_> = reader
- .build()
- .unwrap()
- .collect::<ArrowResult<Vec<_>>>()
- .unwrap();
- assert_eq!(batches.len(), 0);
- }
-
#[test]
fn test_page_stats_not_written_by_default() {
let string_field = Field::new("a", DataType::Utf8, false);
diff --git a/parquet/tests/arrow_writer/roundtrip.rs
b/parquet/tests/arrow_writer/roundtrip.rs
index 2e0da18580..9a122fabe9 100644
--- a/parquet/tests/arrow_writer/roundtrip.rs
+++ b/parquet/tests/arrow_writer/roundtrip.rs
@@ -25,9 +25,10 @@ use std::collections::HashMap;
use std::sync::Arc;
use arrow::datatypes::ToByteSlice;
+use arrow::error::Result as ArrowResult;
use arrow_array::builder::{
- FixedSizeBinaryBuilder, ListBuilder, PrimitiveDictionaryBuilder,
PrimitiveRunBuilder,
- StringViewBuilder,
+ FixedSizeBinaryBuilder, Int32Builder, ListBuilder,
PrimitiveDictionaryBuilder,
+ PrimitiveRunBuilder, StringViewBuilder, StructBuilder,
};
use arrow_array::cast::AsArray;
use arrow_array::types::{
@@ -42,25 +43,26 @@ use arrow_array::{
DurationSecondArray, FixedSizeBinaryArray, Float16Array, Float32Array,
Float64Array, Int8Array,
Int16Array, Int32Array, Int32DictionaryArray, Int32RunArray, Int64Array,
IntervalDayTimeArray,
IntervalYearMonthArray, LargeBinaryArray, LargeListArray,
LargeListViewArray, LargeStringArray,
- ListArray, ListViewArray, NullArray, PrimitiveArray, RecordBatch,
RecordBatchReader, RunArray,
- StringArray, StringViewArray, StructArray, Time32MillisecondArray,
Time32SecondArray,
- Time64MicrosecondArray, Time64NanosecondArray, TimestampMicrosecondArray,
+ ListArray, ListViewArray, NullArray, PrimitiveArray, RecordBatch,
RecordBatchOptions,
+ RecordBatchReader, RunArray, StringArray, StringViewArray, StructArray,
Time32MillisecondArray,
+ Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray,
TimestampMicrosecondArray,
TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray,
UInt8Array,
UInt8DictionaryArray, UInt16Array, UInt32Array, UInt64Array,
};
-use arrow_buffer::{ArrowNativeType, Buffer, IntervalDayTime, NullBuffer, i256};
+use arrow_buffer::{ArrowNativeType, Buffer, IntervalDayTime, NullBuffer,
OffsetBuffer, i256};
use arrow_data::{ArrayData, ArrayDataBuilder};
use arrow_schema::{DataType as ArrowDataType, Field, Fields, Schema,
SchemaRef, TimeUnit};
use bytes::Bytes;
use half::f16;
use num_traits::{FromPrimitive, PrimInt, ToPrimitive};
-use parquet::arrow::ArrowWriter;
use parquet::arrow::arrow_reader::{
ArrowReaderOptions, ParquetRecordBatchReader,
ParquetRecordBatchReaderBuilder,
};
+use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY};
use parquet::basic::Type as PhysicalType;
use parquet::errors::Result;
use parquet::file::properties::WriterProperties;
+use tempfile::tempfile;
#[test]
#[cfg_attr(miri, ignore)] // Takes too long
@@ -1622,6 +1624,381 @@ fn interval_day_time_single_column() {
]);
}
+#[test]
+fn arrow_writer_test_type_compatibility() {
+ fn ensure_compatible_write<T1, T2>(array1: T1, array2: T2,
expected_result: T1)
+ where
+ T1: Array + 'static,
+ T2: Array + 'static,
+ {
+ let schema1 = Arc::new(Schema::new(vec![Field::new(
+ "a",
+ array1.data_type().clone(),
+ false,
+ )]));
+
+ let file = tempfile().unwrap();
+ let mut writer =
+ ArrowWriter::try_new(file.try_clone().unwrap(), schema1.clone(),
None).unwrap();
+
+ let rb1 = RecordBatch::try_new(schema1.clone(),
vec![Arc::new(array1)]).unwrap();
+ writer.write(&rb1).unwrap();
+
+ let schema2 = Arc::new(Schema::new(vec![Field::new(
+ "a",
+ array2.data_type().clone(),
+ false,
+ )]));
+ let rb2 = RecordBatch::try_new(schema2,
vec![Arc::new(array2)]).unwrap();
+ writer.write(&rb2).unwrap();
+
+ writer.close().unwrap();
+
+ let mut record_batch_reader =
+ ParquetRecordBatchReader::try_new(file.try_clone().unwrap(),
1024).unwrap();
+ let actual_batch = record_batch_reader.next().unwrap().unwrap();
+
+ let expected_batch =
+ RecordBatch::try_new(schema1,
vec![Arc::new(expected_result)]).unwrap();
+ assert_eq!(actual_batch, expected_batch);
+ }
+
+ // check compatibility between native and dictionaries
+
+ ensure_compatible_write(
+ DictionaryArray::new(
+ UInt8Array::from_iter_values(vec![0]),
+ Arc::new(StringArray::from_iter_values(vec!["parquet"])),
+ ),
+ StringArray::from_iter_values(vec!["barquet"]),
+ DictionaryArray::new(
+ UInt8Array::from_iter_values(vec![0, 1]),
+ Arc::new(StringArray::from_iter_values(vec!["parquet",
"barquet"])),
+ ),
+ );
+
+ ensure_compatible_write(
+ StringArray::from_iter_values(vec!["parquet"]),
+ DictionaryArray::new(
+ UInt8Array::from_iter_values(vec![0]),
+ Arc::new(StringArray::from_iter_values(vec!["barquet"])),
+ ),
+ StringArray::from_iter_values(vec!["parquet", "barquet"]),
+ );
+
+ // check compatibility between dictionaries with different key types
+
+ ensure_compatible_write(
+ DictionaryArray::new(
+ UInt8Array::from_iter_values(vec![0]),
+ Arc::new(StringArray::from_iter_values(vec!["parquet"])),
+ ),
+ DictionaryArray::new(
+ UInt16Array::from_iter_values(vec![0]),
+ Arc::new(StringArray::from_iter_values(vec!["barquet"])),
+ ),
+ DictionaryArray::new(
+ UInt8Array::from_iter_values(vec![0, 1]),
+ Arc::new(StringArray::from_iter_values(vec!["parquet",
"barquet"])),
+ ),
+ );
+
+ // check compatibility between dictionaries with different value types
+ ensure_compatible_write(
+ DictionaryArray::new(
+ UInt8Array::from_iter_values(vec![0]),
+ Arc::new(StringArray::from_iter_values(vec!["parquet"])),
+ ),
+ DictionaryArray::new(
+ UInt8Array::from_iter_values(vec![0]),
+ Arc::new(LargeStringArray::from_iter_values(vec!["barquet"])),
+ ),
+ DictionaryArray::new(
+ UInt8Array::from_iter_values(vec![0, 1]),
+ Arc::new(StringArray::from_iter_values(vec!["parquet",
"barquet"])),
+ ),
+ );
+
+ // check compatibility between a dictionary and a native array with a
different type
+ ensure_compatible_write(
+ DictionaryArray::new(
+ UInt8Array::from_iter_values(vec![0]),
+ Arc::new(StringArray::from_iter_values(vec!["parquet"])),
+ ),
+ LargeStringArray::from_iter_values(vec!["barquet"]),
+ DictionaryArray::new(
+ UInt8Array::from_iter_values(vec![0, 1]),
+ Arc::new(StringArray::from_iter_values(vec!["parquet",
"barquet"])),
+ ),
+ );
+
+ // check compatibility for string types
+
+ ensure_compatible_write(
+ StringArray::from_iter_values(vec!["parquet"]),
+ LargeStringArray::from_iter_values(vec!["barquet"]),
+ StringArray::from_iter_values(vec!["parquet", "barquet"]),
+ );
+
+ ensure_compatible_write(
+ LargeStringArray::from_iter_values(vec!["parquet"]),
+ StringArray::from_iter_values(vec!["barquet"]),
+ LargeStringArray::from_iter_values(vec!["parquet", "barquet"]),
+ );
+
+ ensure_compatible_write(
+ StringArray::from_iter_values(vec!["parquet"]),
+ StringViewArray::from_iter_values(vec!["barquet"]),
+ StringArray::from_iter_values(vec!["parquet", "barquet"]),
+ );
+
+ ensure_compatible_write(
+ StringViewArray::from_iter_values(vec!["parquet"]),
+ StringArray::from_iter_values(vec!["barquet"]),
+ StringViewArray::from_iter_values(vec!["parquet", "barquet"]),
+ );
+
+ ensure_compatible_write(
+ LargeStringArray::from_iter_values(vec!["parquet"]),
+ StringViewArray::from_iter_values(vec!["barquet"]),
+ LargeStringArray::from_iter_values(vec!["parquet", "barquet"]),
+ );
+
+ ensure_compatible_write(
+ StringViewArray::from_iter_values(vec!["parquet"]),
+ LargeStringArray::from_iter_values(vec!["barquet"]),
+ StringViewArray::from_iter_values(vec!["parquet", "barquet"]),
+ );
+
+ // check compatibility for binary types
+
+ ensure_compatible_write(
+ BinaryArray::from_iter_values(vec![b"parquet"]),
+ LargeBinaryArray::from_iter_values(vec![b"barquet"]),
+ BinaryArray::from_iter_values(vec![b"parquet", b"barquet"]),
+ );
+
+ ensure_compatible_write(
+ LargeBinaryArray::from_iter_values(vec![b"parquet"]),
+ BinaryArray::from_iter_values(vec![b"barquet"]),
+ LargeBinaryArray::from_iter_values(vec![b"parquet", b"barquet"]),
+ );
+
+ ensure_compatible_write(
+ BinaryArray::from_iter_values(vec![b"parquet"]),
+ BinaryViewArray::from_iter_values(vec![b"barquet"]),
+ BinaryArray::from_iter_values(vec![b"parquet", b"barquet"]),
+ );
+
+ ensure_compatible_write(
+ BinaryViewArray::from_iter_values(vec![b"parquet"]),
+ BinaryArray::from_iter_values(vec![b"barquet"]),
+ BinaryViewArray::from_iter_values(vec![b"parquet", b"barquet"]),
+ );
+
+ ensure_compatible_write(
+ BinaryViewArray::from_iter_values(vec![b"parquet"]),
+ LargeBinaryArray::from_iter_values(vec![b"barquet"]),
+ BinaryViewArray::from_iter_values(vec![b"parquet", b"barquet"]),
+ );
+
+ ensure_compatible_write(
+ LargeBinaryArray::from_iter_values(vec![b"parquet"]),
+ BinaryViewArray::from_iter_values(vec![b"barquet"]),
+ LargeBinaryArray::from_iter_values(vec![b"parquet", b"barquet"]),
+ );
+
+ // check compatibility for list types
+
+ let list_field_metadata = HashMap::from_iter(vec![(
+ PARQUET_FIELD_ID_META_KEY.to_string(),
+ "1".to_string(),
+ )]);
+ let list_field = Field::new_list_field(ArrowDataType::Int32, false);
+
+ let values1 = Arc::new(Int32Array::from(vec![0, 1, 2, 3, 4]));
+ let offsets1 = OffsetBuffer::new(vec![0, 2, 5].into());
+
+ let values2 = Arc::new(Int32Array::from(vec![5, 6, 7, 8, 9]));
+ let offsets2 = OffsetBuffer::new(vec![0, 3, 5].into());
+
+ let values_expected = Arc::new(Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6,
7, 8, 9]));
+ let offsets_expected = OffsetBuffer::new(vec![0, 2, 5, 8, 10].into());
+
+ ensure_compatible_write(
+ // when the initial schema has the metadata ...
+ ListArray::try_new(
+ Arc::new(
+ list_field
+ .clone()
+ .with_metadata(list_field_metadata.clone()),
+ ),
+ offsets1,
+ values1,
+ None,
+ )
+ .unwrap(),
+ // ... and some intermediate schema doesn't have the metadata
+ ListArray::try_new(Arc::new(list_field.clone()), offsets2, values2,
None).unwrap(),
+ // ... the write will still go through, and the resulting schema will
inherit the initial metadata
+ ListArray::try_new(
+ Arc::new(
+ list_field
+ .clone()
+ .with_metadata(list_field_metadata.clone()),
+ ),
+ offsets_expected,
+ values_expected,
+ None,
+ )
+ .unwrap(),
+ );
+}
+
+#[test]
+#[cfg_attr(miri, ignore)] // Takes too long
+fn test_list_of_struct_roundtrip() {
+ // define schema
+ let int_field = Field::new("a", ArrowDataType::Int32, true);
+ let int_field2 = Field::new("b", ArrowDataType::Int32, true);
+
+ let int_builder = Int32Builder::with_capacity(10);
+ let int_builder2 = Int32Builder::with_capacity(10);
+
+ let struct_builder = StructBuilder::new(
+ vec![int_field, int_field2],
+ vec![Box::new(int_builder), Box::new(int_builder2)],
+ );
+ let mut list_builder = ListBuilder::new(struct_builder);
+
+ // Construct the following array
+ // [{a: 1, b: 2}], [], null, [null, null], [{a: null, b: 3}], [{a: 2, b:
null}]
+
+ // [{a: 1, b: 2}]
+ let values = list_builder.values();
+ values
+ .field_builder::<Int32Builder>(0)
+ .unwrap()
+ .append_value(1);
+ values
+ .field_builder::<Int32Builder>(1)
+ .unwrap()
+ .append_value(2);
+ values.append(true);
+ list_builder.append(true);
+
+ // []
+ list_builder.append(true);
+
+ // null
+ list_builder.append(false);
+
+ // [null, null]
+ let values = list_builder.values();
+ values
+ .field_builder::<Int32Builder>(0)
+ .unwrap()
+ .append_null();
+ values
+ .field_builder::<Int32Builder>(1)
+ .unwrap()
+ .append_null();
+ values.append(false);
+ values
+ .field_builder::<Int32Builder>(0)
+ .unwrap()
+ .append_null();
+ values
+ .field_builder::<Int32Builder>(1)
+ .unwrap()
+ .append_null();
+ values.append(false);
+ list_builder.append(true);
+
+ // [{a: null, b: 3}]
+ let values = list_builder.values();
+ values
+ .field_builder::<Int32Builder>(0)
+ .unwrap()
+ .append_null();
+ values
+ .field_builder::<Int32Builder>(1)
+ .unwrap()
+ .append_value(3);
+ values.append(true);
+ list_builder.append(true);
+
+ // [{a: 2, b: null}]
+ let values = list_builder.values();
+ values
+ .field_builder::<Int32Builder>(0)
+ .unwrap()
+ .append_value(2);
+ values
+ .field_builder::<Int32Builder>(1)
+ .unwrap()
+ .append_null();
+ values.append(true);
+ list_builder.append(true);
+
+ let array = Arc::new(list_builder.finish());
+
+ RoundTripTest::new(array).run();
+}
+
+#[test]
+// https://github.com/apache/arrow-rs/issues/6988
+fn test_roundtrip_empty_schema() {
+ // create empty record batch with empty schema
+ let empty_batch = RecordBatch::try_new_with_options(
+ Arc::new(Schema::empty()),
+ vec![],
+ &RecordBatchOptions::default().with_row_count(Some(0)),
+ )
+ .unwrap();
+
+ // write to parquet
+ let mut parquet_bytes: Vec<u8> = Vec::new();
+ let mut writer = ArrowWriter::try_new(&mut parquet_bytes,
empty_batch.schema(), None).unwrap();
+ writer.write(&empty_batch).unwrap();
+ writer.close().unwrap();
+
+ // read from parquet
+ let bytes = Bytes::from(parquet_bytes);
+ let reader = ParquetRecordBatchReaderBuilder::try_new(bytes).unwrap();
+ assert_eq!(reader.schema(), &empty_batch.schema());
+ let batches: Vec<_> = reader
+ .build()
+ .unwrap()
+ .collect::<ArrowResult<Vec<_>>>()
+ .unwrap();
+ assert_eq!(batches.len(), 0);
+}
+
+#[test]
+fn test_arrow_writer_nullable() {
+ let batch_schema = Schema::new(vec![Field::new("int32",
ArrowDataType::Int32, false)]);
+ let file_schema = Schema::new(vec![Field::new("int32",
ArrowDataType::Int32, true)]);
+ let file_schema = Arc::new(file_schema);
+
+ let batch = RecordBatch::try_new(
+ Arc::new(batch_schema),
+ vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4])) as _],
+ )
+ .unwrap();
+
+ let mut buf = Vec::with_capacity(1024);
+ let mut writer = ArrowWriter::try_new(&mut buf, file_schema.clone(),
None).unwrap();
+ writer.write(&batch).unwrap();
+ writer.close().unwrap();
+
+ let mut read = ParquetRecordBatchReader::try_new(Bytes::from(buf),
1024).unwrap();
+ let back = read.next().unwrap().unwrap();
+ assert_eq!(back.schema(), file_schema);
+ assert_ne!(back.schema(), batch.schema());
+ assert_eq!(back.column(0).as_ref(), batch.column(0).as_ref());
+}
+
#[test]
fn test_unsigned_roundtrip() {
let schema = Arc::new(Schema::new(vec