kosiew commented on code in PR #27056:
URL: https://github.com/apache/datafusion/pull/27056#discussion_r4641868494
##########
datafusion/proto-common/src/from_proto/mod.rs:
##########
@@ -1921,4 +1963,178 @@ mod tests {
let recovered = table_parquet_options_proto_round_trip(opts.clone());
assert!(!recovered.global.content_defined_chunking.enabled);
}
+
+ #[test]
+ fn decimal_value_of_wrong_length_is_an_error() {
+ use datafusion_common::ScalarValue;
+ use datafusion_common::arrow::datatypes::i256;
+
+ use crate::protobuf_common::scalar_value::Value;
+
+ for (scalar, expected) in [
+ (
+ ScalarValue::Decimal32(Some(123), 9, 2),
+ "General error: Expected 4 bytes for Decimal32Value but got 3",
+ ),
+ (
+ ScalarValue::Decimal64(Some(123), 18, 2),
+ "General error: Expected 8 bytes for Decimal64Value but got 7",
+ ),
+ (
+ ScalarValue::Decimal128(Some(123), 38, 2),
+ "General error: Expected 16 bytes for Decimal128Value but got
15",
+ ),
+ (
+ ScalarValue::Decimal256(Some(i256::from(123)), 76, 2),
+ "General error: Expected 32 bytes for Decimal256Value but got
31",
+ ),
+ ] {
+ let mut proto =
+
crate::protobuf_common::ScalarValue::try_from(&scalar).unwrap();
+ assert_eq!(ScalarValue::try_from(&proto).unwrap(), scalar);
+
+ let bytes = match proto.value.as_mut().unwrap() {
+ Value::Decimal32Value(v) => &mut v.value,
+ Value::Decimal64Value(v) => &mut v.value,
+ Value::Decimal128Value(v) => &mut v.value,
+ Value::Decimal256Value(v) => &mut v.value,
+ other => panic!("unexpected encoding {other:?}"),
+ };
+ bytes.pop();
+ let err = ScalarValue::try_from(&proto).unwrap_err();
+ assert_eq!(err.to_string(), expected);
+ }
+ }
+
+ #[test]
+ fn nested_value_whose_batch_disagrees_with_its_kind_is_an_error() {
+ use datafusion_common::ScalarValue;
+ use datafusion_common::arrow::datatypes::{DataType, Field};
+ use datafusion_common::scalar::ScalarStructBuilder;
+
+ use crate::protobuf_common::scalar_value::Value;
+
+ let encode = |scalar: &ScalarValue| {
+ let proto =
crate::protobuf_common::ScalarValue::try_from(scalar).unwrap();
+ assert_eq!(&ScalarValue::try_from(&proto).unwrap(), scalar);
+ match proto.value {
+ Some(Value::ListValue(v) | Value::StructValue(v)) => v,
+ other => panic!("unexpected encoding {other:?}"),
+ }
+ };
+ let list = encode(&ScalarValue::List(ScalarValue::new_list_nullable(
+ &[ScalarValue::Int32(Some(1))],
+ &DataType::Int32,
+ )));
+ let strct = encode(
+ &ScalarStructBuilder::new()
+ .with_scalar(
+ Field::new("a", DataType::Int32, false),
+ ScalarValue::Int32(Some(1)),
+ )
+ .build()
+ .unwrap(),
+ );
+
+ for (value, kind) in [
+ (Value::ListValue(strct), "ListValue"),
+ (Value::LargeListValue(list.clone()), "LargeListValue"),
+ (
+ Value::FixedSizeListValue(list.clone()),
+ "FixedSizeListValue",
+ ),
+ (Value::ListViewValue(list.clone()), "ListViewValue"),
+ (
+ Value::LargeListViewValue(list.clone()),
+ "LargeListViewValue",
+ ),
+ (Value::StructValue(list.clone()), "StructValue"),
+ (Value::MapValue(list.clone()), "MapValue"),
+ ] {
+ let proto = crate::protobuf_common::ScalarValue { value:
Some(value) };
+ let err = ScalarValue::try_from(&proto).unwrap_err().to_string();
+ assert!(
+ err.contains(&format!(
+ "for {kind} while deserializing nested ScalarValue"
+ )),
+ "{err}"
+ );
+ }
+
+ let no_columns = crate::protobuf_common::ScalarNestedValue {
Review Comment:
Could we use matching IPC metadata and payload from a genuine zero-column
batch? The current fixture replaces the schema but retains list IPC data,
relying on Arrow ignoring the extra field nodes and buffers.
##########
datafusion/proto-common/src/from_proto/mod.rs:
##########
@@ -1921,4 +1963,178 @@ mod tests {
let recovered = table_parquet_options_proto_round_trip(opts.clone());
assert!(!recovered.global.content_defined_chunking.enabled);
}
+
+ #[test]
+ fn decimal_value_of_wrong_length_is_an_error() {
+ use datafusion_common::ScalarValue;
+ use datafusion_common::arrow::datatypes::i256;
+
+ use crate::protobuf_common::scalar_value::Value;
+
+ for (scalar, expected) in [
+ (
+ ScalarValue::Decimal32(Some(123), 9, 2),
+ "General error: Expected 4 bytes for Decimal32Value but got 3",
+ ),
+ (
+ ScalarValue::Decimal64(Some(123), 18, 2),
+ "General error: Expected 8 bytes for Decimal64Value but got 7",
+ ),
+ (
+ ScalarValue::Decimal128(Some(123), 38, 2),
+ "General error: Expected 16 bytes for Decimal128Value but got
15",
+ ),
+ (
+ ScalarValue::Decimal256(Some(i256::from(123)), 76, 2),
+ "General error: Expected 32 bytes for Decimal256Value but got
31",
+ ),
+ ] {
+ let mut proto =
+
crate::protobuf_common::ScalarValue::try_from(&scalar).unwrap();
+ assert_eq!(ScalarValue::try_from(&proto).unwrap(), scalar);
+
+ let bytes = match proto.value.as_mut().unwrap() {
+ Value::Decimal32Value(v) => &mut v.value,
+ Value::Decimal64Value(v) => &mut v.value,
+ Value::Decimal128Value(v) => &mut v.value,
+ Value::Decimal256Value(v) => &mut v.value,
+ other => panic!("unexpected encoding {other:?}"),
+ };
+ bytes.pop();
+ let err = ScalarValue::try_from(&proto).unwrap_err();
+ assert_eq!(err.to_string(), expected);
+ }
+ }
+
+ #[test]
+ fn nested_value_whose_batch_disagrees_with_its_kind_is_an_error() {
+ use datafusion_common::ScalarValue;
+ use datafusion_common::arrow::datatypes::{DataType, Field};
+ use datafusion_common::scalar::ScalarStructBuilder;
+
+ use crate::protobuf_common::scalar_value::Value;
+
+ let encode = |scalar: &ScalarValue| {
+ let proto =
crate::protobuf_common::ScalarValue::try_from(scalar).unwrap();
+ assert_eq!(&ScalarValue::try_from(&proto).unwrap(), scalar);
+ match proto.value {
+ Some(Value::ListValue(v) | Value::StructValue(v)) => v,
+ other => panic!("unexpected encoding {other:?}"),
+ }
+ };
+ let list = encode(&ScalarValue::List(ScalarValue::new_list_nullable(
+ &[ScalarValue::Int32(Some(1))],
+ &DataType::Int32,
+ )));
+ let strct = encode(
+ &ScalarStructBuilder::new()
+ .with_scalar(
+ Field::new("a", DataType::Int32, false),
+ ScalarValue::Int32(Some(1)),
+ )
+ .build()
+ .unwrap(),
+ );
+
+ for (value, kind) in [
+ (Value::ListValue(strct), "ListValue"),
+ (Value::LargeListValue(list.clone()), "LargeListValue"),
+ (
+ Value::FixedSizeListValue(list.clone()),
+ "FixedSizeListValue",
+ ),
+ (Value::ListViewValue(list.clone()), "ListViewValue"),
+ (
+ Value::LargeListViewValue(list.clone()),
+ "LargeListViewValue",
+ ),
+ (Value::StructValue(list.clone()), "StructValue"),
+ (Value::MapValue(list.clone()), "MapValue"),
+ ] {
+ let proto = crate::protobuf_common::ScalarValue { value:
Some(value) };
+ let err = ScalarValue::try_from(&proto).unwrap_err().to_string();
+ assert!(
+ err.contains(&format!(
+ "for {kind} while deserializing nested ScalarValue"
+ )),
+ "{err}"
+ );
+ }
+
+ let no_columns = crate::protobuf_common::ScalarNestedValue {
+ schema: Some(crate::protobuf_common::Schema::default()),
+ ..list
+ };
+ let proto = crate::protobuf_common::ScalarValue {
+ value: Some(Value::ListValue(no_columns)),
+ };
+ assert_eq!(
+ ScalarValue::try_from(&proto).unwrap_err().to_string(),
+ "General error: No column in record batch while deserializing
nested ScalarValue"
+ );
+ }
+
+ #[test]
+ fn union_of_more_than_128_fields_without_type_ids_is_an_error() {
+ use datafusion_common::arrow::datatypes::{DataType, Field};
+
+ use crate::protobuf_common::arrow_type::ArrowTypeEnum;
+
+ let union = |children: usize| crate::protobuf_common::ArrowType {
+ arrow_type_enum:
Some(ArrowTypeEnum::Union(crate::protobuf_common::Union {
+ union_types: (0..children)
+ .map(|i| {
+ let field = Field::new(format!("f{i}"),
DataType::Int32, true);
+
crate::protobuf_common::Field::try_from(&field).unwrap()
+ })
+ .collect(),
+ union_mode: crate::protobuf_common::UnionMode::Sparse as i32,
+ type_ids: vec![],
+ })),
+ };
+
+ assert!(matches!(
+ DataType::try_from(&union(128)).unwrap(),
Review Comment:
Consider asserting that fields f0 through f127 retain implicit IDs 0..=127.
The current length assertion would miss reordered children or incorrect ID
assignments.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]