Rich-T-kid commented on code in PR #12161:
URL: https://github.com/apache/arrow-rs/pull/12161#discussion_r4545983879
##########
parquet/src/arrow/arrow_reader/tests/schema.rs:
##########
@@ -622,6 +622,204 @@ fn test_read_binary_as_utf8() {
);
}
+fn dictionary_type(value_type: ArrowDataType) -> ArrowDataType {
+ ArrowDataType::Dictionary(Box::new(ArrowDataType::Int32),
Box::new(value_type))
+}
+
+#[test]
+fn test_binary_column_as_utf8_dictionary() {
+ let file = write_parquet_from_iter(vec![(
+ "binary",
+ Arc::new(BinaryArray::from(vec![
+ b"one".as_ref(),
+ b"two".as_ref(),
+ b"one".as_ref(),
+ ])) as ArrayRef,
+ )]);
+ let data_type = dictionary_type(ArrowDataType::Utf8);
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "binary",
+ data_type.clone(),
+ false,
+ )]));
+ let options = ArrowReaderOptions::new().with_schema(schema);
+ let mut reader =
ParquetRecordBatchReaderBuilder::try_new_with_options(file, options)
+ .unwrap()
+ .build()
+ .unwrap();
+
+ let batch = reader.next().unwrap().unwrap();
+ assert_eq!(batch.column(0).data_type(), &data_type);
+ let dictionary = batch
+ .column(0)
+ .as_dictionary::<arrow::datatypes::Int32Type>();
+ assert_eq!(dictionary.keys().values().as_ref(), &[0, 1, 0]);
+ assert_eq!(
+ dictionary.values().as_string::<i32>(),
+ &StringArray::from(vec!["one", "two"])
+ );
+}
+
+#[test]
+fn test_dictionary_requested_value_type_variants() {
+ let values = vec![b"one".as_ref(), b"two".as_ref(), b"one".as_ref()];
+ let file = write_parquet_from_iter(vec![
+ (
+ "large_utf8",
+ Arc::new(LargeBinaryArray::from(values.clone())) as ArrayRef,
+ ),
+ ("utf8_view", Arc::new(BinaryArray::from(values)) as ArrayRef),
+ ]);
+ let schema = Arc::new(Schema::new(vec![
+ Field::new(
+ "large_utf8",
+ dictionary_type(ArrowDataType::LargeUtf8),
+ false,
+ ),
+ Field::new("utf8_view", dictionary_type(ArrowDataType::Utf8View),
false),
+ ]));
+
+ let options = ArrowReaderOptions::new().with_schema(Arc::clone(&schema));
+ let mut reader =
ParquetRecordBatchReaderBuilder::try_new_with_options(file, options)
+ .unwrap()
+ .build()
+ .unwrap();
+
+ let batch = reader.next().unwrap().unwrap();
+ assert_eq!(batch.schema().as_ref(), schema.as_ref());
+ let expected = StringArray::from(vec!["one", "two", "one"]);
+ for column in batch.columns() {
+ let actual = arrow::compute::cast(column,
&ArrowDataType::Utf8).unwrap();
+ assert_eq!(actual.as_string::<i32>(), &expected);
+ }
+}
+
+#[test]
+fn test_dictionary_value_cache_across_row_groups() {
+ let values = Arc::new(BinaryArray::from(vec![
+ b"one".as_ref(),
+ b"two".as_ref(),
+ b"one".as_ref(),
+ b"two".as_ref(),
+ b"three".as_ref(),
+ b"four".as_ref(),
+ b"three".as_ref(),
+ b"four".as_ref(),
+ ])) as ArrayRef;
+ let batch = RecordBatch::try_from_iter(vec![("binary", values)]).unwrap();
+ let props = WriterProperties::builder()
+ .set_max_row_group_row_count(Some(4))
+ .build();
+ let mut parquet_data = vec![];
+ let mut writer = ArrowWriter::try_new(&mut parquet_data, batch.schema(),
Some(props)).unwrap();
+ writer.write(&batch).unwrap();
+ writer.close().unwrap();
+
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "binary",
+ dictionary_type(ArrowDataType::Utf8),
+ false,
+ )]));
+ let options = ArrowReaderOptions::new().with_schema(Arc::clone(&schema));
+ let reader =
+
ParquetRecordBatchReaderBuilder::try_new_with_options(Bytes::from(parquet_data),
options)
+ .unwrap()
+ .with_batch_size(2)
+ .build()
+ .unwrap();
+
+ let batches = reader.collect::<std::result::Result<Vec<_>, _>>().unwrap();
+ assert_eq!(batches.len(), 4);
+ let dictionary_values = batches
+ .iter()
+ .map(|batch| {
+ Arc::clone(
+ batch
+ .column(0)
+ .as_dictionary::<arrow::datatypes::Int32Type>()
+ .values(),
+ )
+ })
+ .collect::<Vec<_>>();
+ assert!(Arc::ptr_eq(&dictionary_values[0], &dictionary_values[1]));
+ assert!(!Arc::ptr_eq(&dictionary_values[1], &dictionary_values[2]));
+ assert!(Arc::ptr_eq(&dictionary_values[2], &dictionary_values[3]));
+ assert_eq!(
+ dictionary_values[0].as_string::<i32>(),
+ &StringArray::from(vec!["one", "two"])
+ );
+ assert_eq!(
+ dictionary_values[2].as_string::<i32>(),
+ &StringArray::from(vec!["three", "four"])
+ );
+}
+
+#[test]
+fn test_binary_dictionary_rejects_invalid_utf8() {
+ let file = write_parquet_from_iter(vec![(
+ "binary",
+ Arc::new(BinaryArray::from(vec![
+ b"valid".as_ref(),
+ b"\xDE\x00\xFF".as_ref(),
+ b"valid".as_ref(),
+ ])) as ArrayRef,
+ )]);
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "binary",
+ dictionary_type(ArrowDataType::Utf8),
+ false,
+ )]));
+ let options = ArrowReaderOptions::new().with_schema(schema);
+ let mut reader =
ParquetRecordBatchReaderBuilder::try_new_with_options(file, options)
+ .unwrap()
+ .build()
+ .unwrap();
+
+ let error = reader.next().unwrap().unwrap_err();
+ assert!(error.to_string().contains("non UTF-8 data"), "{error}");
+}
+
+#[test]
+fn test_plain_binary_as_utf8_view_dictionary() {
+ let values = Arc::new(BinaryArray::from(vec![
+ b"one".as_ref(),
+ b"two".as_ref(),
+ b"one".as_ref(),
+ ])) as ArrayRef;
+ let batch = RecordBatch::try_from_iter(vec![("binary", values)]).unwrap();
+ let props = WriterProperties::builder()
+ .set_dictionary_enabled(false)
+ .build();
+ let mut parquet_data = vec![];
+ let mut writer = ArrowWriter::try_new(&mut parquet_data, batch.schema(),
Some(props)).unwrap();
+ writer.write(&batch).unwrap();
+ writer.close().unwrap();
+
+ let data_type = dictionary_type(ArrowDataType::Utf8View);
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "binary",
+ data_type.clone(),
+ false,
+ )]));
+ let options = ArrowReaderOptions::new().with_schema(schema);
+ let mut reader =
+
ParquetRecordBatchReaderBuilder::try_new_with_options(Bytes::from(parquet_data),
options)
+ .unwrap()
+ .build()
+ .unwrap();
+
+ let batch = reader.next().unwrap().unwrap();
+ assert_eq!(batch.column(0).data_type(), &data_type);
+ let dictionary = batch
+ .column(0)
+ .as_dictionary::<arrow::datatypes::Int32Type>();
+ assert_eq!(dictionary.keys().values().as_ref(), &[0, 1, 0]);
+ assert_eq!(
+ dictionary.values().as_string_view(),
+ &StringViewArray::from(vec!["one", "two"])
+ );
+}
Review Comment:
thanks for this, will pretty much copy and paste this into the pr
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]