laskoviymishka commented on code in PR #3699:
URL: https://github.com/apache/iceberg-rust/pull/3699#discussion_r4611080530
##########
crates/iceberg/Cargo.toml:
##########
@@ -71,6 +71,7 @@ parquet = { workspace = true, features = [
"encryption",
"arrow_canonical_extension_types",
"geospatial",
+ "variant_experimental",
Review Comment:
Should `variant_experimental` sit behind an `iceberg` cargo feature rather
than always on? Through feature unification it turns on for every `parquet` in
a dependent's build, pulls in three `parquet-variant*` crates, and tags
`VARIANT` groups with `arrow.parquet.variant` in schemas other `parquet` users
infer
([extension.rs](https://github.com/apache/arrow-rs/blob/b391e5af7dff9f6909af45c01f7dddb68ba08c0f/parquet/src/arrow/schema/extension.rs#L49-L55)).
The data type is unchanged, and our own reader already tags projected variants.
##########
crates/iceberg/src/writer/file_writer/parquet_writer.rs:
##########
@@ -1061,18 +1089,246 @@ mod tests {
}
#[test]
- fn test_index_by_parquet_path_variant_is_unsupported() {
- // Writing variant columns to Parquet is not supported yet; indexing a
schema that
- // contains one must error rather than silently miss-map columns.
+ fn test_index_by_parquet_path_variant() {
+ // Both leaves of a variant group map to the id of the variant field,
at the top level
+ // and inside a struct, a list and a map.
let schema = Schema::builder()
.with_fields(vec![
NestedField::optional(1, "v",
Type::Variant(VariantType)).into(),
+ NestedField::optional(
+ 2,
+ "s",
+ Type::Struct(StructType::new(vec![
+ NestedField::required(3, "sv",
Type::Variant(VariantType)).into(),
+ ])),
+ )
+ .into(),
+ NestedField::optional(
+ 4,
+ "l",
+ Type::List(ListType::new(
+ NestedField::list_element(5,
Type::Variant(VariantType), true).into(),
+ )),
+ )
+ .into(),
+ NestedField::optional(
+ 6,
+ "m",
+ Type::Map(MapType::new(
+ NestedField::map_key_element(7,
Type::Primitive(PrimitiveType::String))
+ .into(),
+ NestedField::map_value_element(8,
Type::Variant(VariantType), true).into(),
+ )),
+ )
+ .into(),
])
.build()
.unwrap();
let mut visitor = IndexByParquetPathName::new();
- let err = visit_schema(&schema, &mut visitor).unwrap_err();
- assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}");
+ visit_schema(&schema, &mut visitor).unwrap();
+ let expect = HashMap::from([
+ ("v.metadata".to_string(), 1),
+ ("v.value".to_string(), 1),
+ ("s.sv.metadata".to_string(), 3),
+ ("s.sv.value".to_string(), 3),
+ ("l.list.element.metadata".to_string(), 5),
+ ("l.list.element.value".to_string(), 5),
+ ("m.key_value.key".to_string(), 7),
+ ("m.key_value.value.metadata".to_string(), 8),
+ ("m.key_value.value.value".to_string(), 8),
+ ]);
+ assert_eq!(visitor.name_to_id, expect);
+ assert!(visitor.is_variant_value("v.value"));
+ assert!(visitor.is_variant_value("m.key_value.value.value"));
+ assert!(!visitor.is_variant_value("v.metadata"));
+ assert!(!visitor.is_variant_value("m.key_value.key"));
+ }
+
+ /// Makes one variant from each JSON row in the Arrow layout of the
Iceberg table type.
+ /// `None` makes a null row.
+ fn variants_from_json(rows: &[Option<&str>]) -> ArrayRef {
+ let json = Arc::new(arrow_array::StringArray::from(rows.to_vec())) as
ArrayRef;
+ let variants =
ArrayRef::from(parquet::variant::json_to_variant(&json).unwrap());
+ let table_type =
crate::arrow::type_to_arrow_type(&Type::Variant(VariantType)).unwrap();
+ arrow_cast::cast(&variants, &table_type).unwrap()
+ }
+
+ /// Renders every row of a variant column as JSON. A null row renders as
`None`.
+ fn variants_to_json(column: &ArrayRef) -> Vec<Option<String>> {
+ let variants =
parquet::variant::VariantArray::try_new(column.as_ref()).unwrap();
+ let json =
parquet::variant::variant_to_json(&ArrayRef::from(variants)).unwrap();
+ (0..json.len())
+ .map(|i| (!json.is_null(i)).then(|| json.value(i).to_string()))
+ .collect()
+ }
+
+ /// Writes an unshredded variant column, checks the Parquet schema and the
data file
+ /// metrics, and reads the rows back through the Iceberg reader.
+ #[tokio::test]
+ async fn test_parquet_writer_variant_roundtrip() -> Result<()> {
+ let temp_dir = TempDir::new().unwrap();
+ let file_io = FileIO::new_with_fs();
+ let schema = Arc::new(
+ Schema::builder()
+ .with_fields(vec![
+ NestedField::required(1, "id",
Type::Primitive(PrimitiveType::Int)).into(),
+ NestedField::optional(2, "v",
Type::Variant(VariantType)).into(),
Review Comment:
Could the roundtrip also write a `list<variant>` and a `map<string,
variant>` and assert the metrics? The path-index test pins the keys, but if the
writer's leaf paths ever drift from them, the metrics loop skips those leaves
without an error. A version of this against the head commit passes, with value
counts and column sizes present for both fields.
##########
crates/iceberg/src/writer/file_writer/parquet_writer.rs:
##########
@@ -1061,18 +1089,246 @@ mod tests {
}
#[test]
- fn test_index_by_parquet_path_variant_is_unsupported() {
- // Writing variant columns to Parquet is not supported yet; indexing a
schema that
- // contains one must error rather than silently miss-map columns.
+ fn test_index_by_parquet_path_variant() {
+ // Both leaves of a variant group map to the id of the variant field,
at the top level
+ // and inside a struct, a list and a map.
let schema = Schema::builder()
.with_fields(vec![
NestedField::optional(1, "v",
Type::Variant(VariantType)).into(),
+ NestedField::optional(
+ 2,
+ "s",
+ Type::Struct(StructType::new(vec![
+ NestedField::required(3, "sv",
Type::Variant(VariantType)).into(),
+ ])),
+ )
+ .into(),
+ NestedField::optional(
+ 4,
+ "l",
+ Type::List(ListType::new(
+ NestedField::list_element(5,
Type::Variant(VariantType), true).into(),
+ )),
+ )
+ .into(),
+ NestedField::optional(
+ 6,
+ "m",
+ Type::Map(MapType::new(
+ NestedField::map_key_element(7,
Type::Primitive(PrimitiveType::String))
+ .into(),
+ NestedField::map_value_element(8,
Type::Variant(VariantType), true).into(),
+ )),
+ )
+ .into(),
])
.build()
.unwrap();
let mut visitor = IndexByParquetPathName::new();
- let err = visit_schema(&schema, &mut visitor).unwrap_err();
- assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}");
+ visit_schema(&schema, &mut visitor).unwrap();
+ let expect = HashMap::from([
+ ("v.metadata".to_string(), 1),
+ ("v.value".to_string(), 1),
+ ("s.sv.metadata".to_string(), 3),
+ ("s.sv.value".to_string(), 3),
+ ("l.list.element.metadata".to_string(), 5),
+ ("l.list.element.value".to_string(), 5),
+ ("m.key_value.key".to_string(), 7),
+ ("m.key_value.value.metadata".to_string(), 8),
+ ("m.key_value.value.value".to_string(), 8),
+ ]);
+ assert_eq!(visitor.name_to_id, expect);
+ assert!(visitor.is_variant_value("v.value"));
+ assert!(visitor.is_variant_value("m.key_value.value.value"));
+ assert!(!visitor.is_variant_value("v.metadata"));
+ assert!(!visitor.is_variant_value("m.key_value.key"));
+ }
+
+ /// Makes one variant from each JSON row in the Arrow layout of the
Iceberg table type.
+ /// `None` makes a null row.
+ fn variants_from_json(rows: &[Option<&str>]) -> ArrayRef {
+ let json = Arc::new(arrow_array::StringArray::from(rows.to_vec())) as
ArrayRef;
+ let variants =
ArrayRef::from(parquet::variant::json_to_variant(&json).unwrap());
+ let table_type =
crate::arrow::type_to_arrow_type(&Type::Variant(VariantType)).unwrap();
+ arrow_cast::cast(&variants, &table_type).unwrap()
+ }
+
+ /// Renders every row of a variant column as JSON. A null row renders as
`None`.
+ fn variants_to_json(column: &ArrayRef) -> Vec<Option<String>> {
+ let variants =
parquet::variant::VariantArray::try_new(column.as_ref()).unwrap();
+ let json =
parquet::variant::variant_to_json(&ArrayRef::from(variants)).unwrap();
+ (0..json.len())
+ .map(|i| (!json.is_null(i)).then(|| json.value(i).to_string()))
+ .collect()
+ }
+
+ /// Writes an unshredded variant column, checks the Parquet schema and the
data file
+ /// metrics, and reads the rows back through the Iceberg reader.
+ #[tokio::test]
+ async fn test_parquet_writer_variant_roundtrip() -> Result<()> {
+ let temp_dir = TempDir::new().unwrap();
+ let file_io = FileIO::new_with_fs();
+ let schema = Arc::new(
+ Schema::builder()
+ .with_fields(vec![
+ NestedField::required(1, "id",
Type::Primitive(PrimitiveType::Int)).into(),
+ NestedField::optional(2, "v",
Type::Variant(VariantType)).into(),
+ NestedField::optional(
+ 3,
+ "s",
+ Type::Struct(StructType::new(vec![
+ NestedField::required(4, "sv",
Type::Variant(VariantType)).into(),
+ ])),
+ )
+ .into(),
+ ])
+ .build()?,
+ );
+ let arrow_schema = Arc::new(schema_to_arrow_schema(&schema)?);
+ let rows = [
+ Some(r#"{"a":1,"b":"x"}"#),
+ Some("[1,2,3]"),
+ None,
+ Some("null"),
+ Some(r#""text""#),
+ Some("-123456789012"),
+ ];
+ let v = variants_from_json(&rows);
+ let sv = variants_from_json(&[
+ Some("1"),
+ Some("2"),
+ Some("3"),
+ Some("4"),
+ Some("5"),
+ Some("6"),
+ ]);
+ let DataType::Struct(s_fields) = arrow_schema.field(2).data_type()
else {
+ unreachable!()
+ };
+ let s = Arc::new(StructArray::new(s_fields.clone(), vec![sv], None))
as ArrayRef;
+ let batch = RecordBatch::try_new(arrow_schema.clone(), vec![
+ Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5, 6])),
+ v.clone(),
+ s,
+ ])?;
+
+ let output =
file_io.new_output(temp_dir.path().join("v.parquet").to_str().unwrap())?;
+ let props = WriterProperties::builder()
+ .set_max_row_group_row_count(Some(4))
+ .build();
+ let mut writer = ParquetWriterBuilder::new(props, schema.clone())
+ .build(output)
+ .await?;
+ writer.write(&batch).await?;
+ let data_file = writer
+ .close()
+ .await?
+ .into_iter()
+ .next()
+ .unwrap()
+ .content(DataContentType::Data)
+ .partition(Struct::empty())
+ .partition_spec_id(0)
+ .build()
+ .unwrap();
+
+ // Metrics: the counts of a variant come from its `metadata` leaf, it
has no bounds, and
+ // its column size covers both leaves.
+ assert_eq!(data_file.record_count(), 6);
+ assert_eq!(
+ *data_file.value_counts(),
+ HashMap::from([(1, 6), (2, 6), (4, 6)])
+ );
+ assert_eq!(
+ *data_file.null_value_counts(),
+ HashMap::from([(1, 0), (2, 1), (4, 0)])
+ );
+ assert_eq!(
+ *data_file.lower_bounds(),
+ HashMap::from([(1, Datum::int(1))])
+ );
+ assert_eq!(
+ *data_file.upper_bounds(),
+ HashMap::from([(1, Datum::int(6))])
+ );
+ assert!(data_file.column_sizes()[&2] > data_file.column_sizes()[&4]);
Review Comment:
Could this pin the size to both leaves? Comparing `v` with `s.sv` doesn't
tie it to the two chunks the comment above describes. Summing
`compressed_size()` of the `v.metadata` and `v.value` chunks from the file
metadata and `assert_eq!`ing that against `column_sizes()[&2]` would.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]