adriangb commented on code in PR #12161:
URL: https://github.com/apache/arrow-rs/pull/12161#discussion_r4543759723
##########
parquet/src/arrow/arrow_reader/tests/schema.rs:
##########
@@ -622,6 +622,204 @@ fn test_read_binary_as_utf8() {
);
}
+fn dictionary_type(value_type: ArrowDataType) -> ArrowDataType {
+ ArrowDataType::Dictionary(Box::new(ArrowDataType::Int32),
Box::new(value_type))
+}
+
+#[test]
+fn test_binary_column_as_utf8_dictionary() {
+ let file = write_parquet_from_iter(vec![(
+ "binary",
+ Arc::new(BinaryArray::from(vec![
+ b"one".as_ref(),
+ b"two".as_ref(),
+ b"one".as_ref(),
+ ])) as ArrayRef,
+ )]);
+ let data_type = dictionary_type(ArrowDataType::Utf8);
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "binary",
+ data_type.clone(),
+ false,
+ )]));
+ let options = ArrowReaderOptions::new().with_schema(schema);
+ let mut reader =
ParquetRecordBatchReaderBuilder::try_new_with_options(file, options)
+ .unwrap()
+ .build()
+ .unwrap();
+
+ let batch = reader.next().unwrap().unwrap();
+ assert_eq!(batch.column(0).data_type(), &data_type);
+ let dictionary = batch
+ .column(0)
+ .as_dictionary::<arrow::datatypes::Int32Type>();
+ assert_eq!(dictionary.keys().values().as_ref(), &[0, 1, 0]);
+ assert_eq!(
+ dictionary.values().as_string::<i32>(),
+ &StringArray::from(vec!["one", "two"])
+ );
+}
+
+#[test]
+fn test_dictionary_requested_value_type_variants() {
+ let values = vec![b"one".as_ref(), b"two".as_ref(), b"one".as_ref()];
+ let file = write_parquet_from_iter(vec![
+ (
+ "large_utf8",
+ Arc::new(LargeBinaryArray::from(values.clone())) as ArrayRef,
+ ),
+ ("utf8_view", Arc::new(BinaryArray::from(values)) as ArrayRef),
+ ]);
+ let schema = Arc::new(Schema::new(vec![
+ Field::new(
+ "large_utf8",
+ dictionary_type(ArrowDataType::LargeUtf8),
+ false,
+ ),
+ Field::new("utf8_view", dictionary_type(ArrowDataType::Utf8View),
false),
+ ]));
+
+ let options = ArrowReaderOptions::new().with_schema(Arc::clone(&schema));
+ let mut reader =
ParquetRecordBatchReaderBuilder::try_new_with_options(file, options)
+ .unwrap()
+ .build()
+ .unwrap();
+
+ let batch = reader.next().unwrap().unwrap();
+ assert_eq!(batch.schema().as_ref(), schema.as_ref());
+ let expected = StringArray::from(vec!["one", "two", "one"]);
+ for column in batch.columns() {
+ let actual = arrow::compute::cast(column,
&ArrowDataType::Utf8).unwrap();
+ assert_eq!(actual.as_string::<i32>(), &expected);
+ }
+}
+
+#[test]
+fn test_dictionary_value_cache_across_row_groups() {
+ let values = Arc::new(BinaryArray::from(vec![
+ b"one".as_ref(),
+ b"two".as_ref(),
+ b"one".as_ref(),
+ b"two".as_ref(),
+ b"three".as_ref(),
+ b"four".as_ref(),
+ b"three".as_ref(),
+ b"four".as_ref(),
+ ])) as ArrayRef;
+ let batch = RecordBatch::try_from_iter(vec![("binary", values)]).unwrap();
+ let props = WriterProperties::builder()
+ .set_max_row_group_row_count(Some(4))
+ .build();
+ let mut parquet_data = vec![];
+ let mut writer = ArrowWriter::try_new(&mut parquet_data, batch.schema(),
Some(props)).unwrap();
+ writer.write(&batch).unwrap();
+ writer.close().unwrap();
+
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "binary",
+ dictionary_type(ArrowDataType::Utf8),
+ false,
+ )]));
+ let options = ArrowReaderOptions::new().with_schema(Arc::clone(&schema));
+ let reader =
+
ParquetRecordBatchReaderBuilder::try_new_with_options(Bytes::from(parquet_data),
options)
+ .unwrap()
+ .with_batch_size(2)
+ .build()
+ .unwrap();
+
+ let batches = reader.collect::<std::result::Result<Vec<_>, _>>().unwrap();
+ assert_eq!(batches.len(), 4);
+ let dictionary_values = batches
+ .iter()
+ .map(|batch| {
+ Arc::clone(
+ batch
+ .column(0)
+ .as_dictionary::<arrow::datatypes::Int32Type>()
+ .values(),
+ )
+ })
+ .collect::<Vec<_>>();
+ assert!(Arc::ptr_eq(&dictionary_values[0], &dictionary_values[1]));
+ assert!(!Arc::ptr_eq(&dictionary_values[1], &dictionary_values[2]));
+ assert!(Arc::ptr_eq(&dictionary_values[2], &dictionary_values[3]));
+ assert_eq!(
+ dictionary_values[0].as_string::<i32>(),
+ &StringArray::from(vec!["one", "two"])
+ );
+ assert_eq!(
+ dictionary_values[2].as_string::<i32>(),
+ &StringArray::from(vec!["three", "four"])
+ );
+}
+
+#[test]
+fn test_binary_dictionary_rejects_invalid_utf8() {
+ let file = write_parquet_from_iter(vec![(
+ "binary",
+ Arc::new(BinaryArray::from(vec![
+ b"valid".as_ref(),
+ b"\xDE\x00\xFF".as_ref(),
+ b"valid".as_ref(),
+ ])) as ArrayRef,
+ )]);
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "binary",
+ dictionary_type(ArrowDataType::Utf8),
+ false,
+ )]));
+ let options = ArrowReaderOptions::new().with_schema(schema);
+ let mut reader =
ParquetRecordBatchReaderBuilder::try_new_with_options(file, options)
+ .unwrap()
+ .build()
+ .unwrap();
+
+ let error = reader.next().unwrap().unwrap_err();
+ assert!(error.to_string().contains("non UTF-8 data"), "{error}");
+}
+
+#[test]
+fn test_plain_binary_as_utf8_view_dictionary() {
+ let values = Arc::new(BinaryArray::from(vec![
+ b"one".as_ref(),
+ b"two".as_ref(),
+ b"one".as_ref(),
+ ])) as ArrayRef;
+ let batch = RecordBatch::try_from_iter(vec![("binary", values)]).unwrap();
+ let props = WriterProperties::builder()
+ .set_dictionary_enabled(false)
+ .build();
+ let mut parquet_data = vec![];
+ let mut writer = ArrowWriter::try_new(&mut parquet_data, batch.schema(),
Some(props)).unwrap();
+ writer.write(&batch).unwrap();
+ writer.close().unwrap();
+
+ let data_type = dictionary_type(ArrowDataType::Utf8View);
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "binary",
+ data_type.clone(),
+ false,
+ )]));
+ let options = ArrowReaderOptions::new().with_schema(schema);
+ let mut reader =
+
ParquetRecordBatchReaderBuilder::try_new_with_options(Bytes::from(parquet_data),
options)
+ .unwrap()
+ .build()
+ .unwrap();
+
+ let batch = reader.next().unwrap().unwrap();
+ assert_eq!(batch.column(0).data_type(), &data_type);
+ let dictionary = batch
+ .column(0)
+ .as_dictionary::<arrow::datatypes::Int32Type>();
+ assert_eq!(dictionary.keys().values().as_ref(), &[0, 1, 0]);
+ assert_eq!(
+ dictionary.values().as_string_view(),
+ &StringViewArray::from(vec!["one", "two"])
+ );
+}
Review Comment:
A regression test for the plain-page case. Without the change above, it
aborts in both debug and release: the reader returns `Ok`, and `unwrap_err`
panics while formatting the invalid string.
```suggestion
}
#[test]
fn test_plain_binary_dictionary_rejects_invalid_utf8() {
let values = Arc::new(BinaryArray::from(vec![
b"valid".as_ref(),
b"\xDE\x00\xFF".as_ref(),
b"valid".as_ref(),
])) as ArrayRef;
let batch = RecordBatch::try_from_iter(vec![("binary",
values)]).unwrap();
let props = WriterProperties::builder()
.set_dictionary_enabled(false)
.build();
let mut parquet_data = vec![];
let mut writer = ArrowWriter::try_new(&mut parquet_data, batch.schema(),
Some(props)).unwrap();
writer.write(&batch).unwrap();
writer.close().unwrap();
let schema = Arc::new(Schema::new(vec![Field::new(
"binary",
dictionary_type(ArrowDataType::Utf8),
false,
)]));
let options = ArrowReaderOptions::new().with_schema(schema);
let mut reader =
ParquetRecordBatchReaderBuilder::try_new_with_options(Bytes::from(parquet_data),
options)
.unwrap()
.build()
.unwrap();
let error = reader.next().unwrap().unwrap_err();
assert!(error.to_string().contains("non UTF-8 data"), "{error}");
}
```
##########
parquet/src/arrow/array_reader/byte_array_dictionary.rs:
##########
@@ -227,13 +247,38 @@ where
let buffer = self.record_reader.consume_record_data();
let null_buffer = self.record_reader.consume_compact_bitmap();
- let array = match &self.buffer_type {
- None => buffer.into_array(null_buffer, &self.data_type, &mut
self.hash_scratch)?,
- Some(buffer_type) => {
+ let ArrowType::Dictionary(_, output_value_type) = &self.data_type else
{
+ unreachable!()
+ };
+ let cache_converted_values = matches!(&buffer, DictionaryBuffer::Dict
{ .. });
+ let convert_values = cache_converted_values
+ || matches!(
+ output_value_type.as_ref(),
+ ArrowType::Utf8View | ArrowType::BinaryView
+ );
+
+ let array = match (&self.buffer_type, convert_values) {
+ (Some(buffer_type), true) => {
Review Comment:
Plain-encoded pages still skip UTF-8 validation for `Dict<_, Utf8>` and
`Dict<_, LargeUtf8>`.
When the writer falls back from dictionary to plain encoding, `buffer` is
`DictionaryBuffer::Values`. `convert_values` is then `false` for any non-view
output type, so the batch goes to `buffer.into_array(null_buffer,
&self.data_type, ...)`. That calls `pack_values_from_offsets_impl`, which
builds the `Utf8` values with `build_unchecked`. The bytes come from an
unannotated BINARY column, so nothing checks them.
If every batch with a `buffer_type` converts, plain pages decode as `Binary`
and go through `try_from_binary`. Only dictionary-page values are cached, so
this does not change the caching behavior.
```suggestion
let cache_converted_values = matches!(&buffer,
DictionaryBuffer::Dict { .. });
// Plain-encoded pages also decode into `buffer_type` and convert,
so that
// `convert_dictionary_values` validates UTF-8 for unannotated
BINARY columns.
let array = match &self.buffer_type {
Some(buffer_type) => {
```
I checked this locally on 12ff11b7a. With this change, the full `parquet`
lib and `arrow_reader` tests pass, and the regression test below passes in
debug and release. I assume `f5ce90324` kept plain-page fallback as on `main`
to limit the scope of this PR. That hole already exists on `main`, so this PR
does not make it worse. If you prefer to keep the scope, I am happy for this to
be a follow-up for #11140 instead.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]