This is an automated email from the ASF dual-hosted git repository.
sdf-jkl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 06411df16f [Variant] Support list paths in ShreddedSchemaBuilder
(#10635)
06411df16f is described below
commit 06411df16f64a5729e2cd00cc57a1aa2dad8cb82
Author: cakeni <[email protected]>
AuthorDate: Wed Sep 16 21:52:46 2026 +0800
[Variant] Support list paths in ShreddedSchemaBuilder (#10635)
# Which issue does this PR close?
- Closes #10615.
# Rationale for this change
`ShreddedSchemaBuilder` can already parse indexed Variant paths, and
core Variant shredding supports lists, but schema construction currently
panics when a path reaches an index. This prevents callers from
describing list element schemas through the builder.
# What changes are included in this PR?
- Add a list node to the builder's intermediate schema tree.
- Treat numeric indexes as references to the shared list element schema,
including for nested lists.
- Document the index behavior and add coverage for list-of-struct and
nested-list schemas.
- Build the existing list shredding test schema through
`ShreddedSchemaBuilder`.
# Are these changes tested?
Yes. `cargo test -p parquet-variant-compute test_variant_schema_builder`
passes (11 tests).
# Are there any user-facing changes?
Yes. Callers can now use indexed paths such as `items[0].id` when
constructing shredding schemas. This is backward compatible.
## AI assistance
OpenAI Codex assisted with implementation, documentation, and test
drafting. I reviewed the resulting design and changes.
---------
Co-authored-by: cakeni <[email protected]>
---
parquet-variant-compute/src/shred_variant.rs | 129 ++++++++++++++++++++++-----
parquet-variant-compute/src/variant_get.rs | 23 +++++
parquet-variant/src/path.rs | 21 ++++-
parquet-variant/src/utils.rs | 4 +-
parquet-variant/src/variant.rs | 1 +
5 files changed, 151 insertions(+), 27 deletions(-)
diff --git a/parquet-variant-compute/src/shred_variant.rs
b/parquet-variant-compute/src/shred_variant.rs
index 6c4af06d51..4db0098f69 100644
--- a/parquet-variant-compute/src/shred_variant.rs
+++ b/parquet-variant-compute/src/shred_variant.rs
@@ -540,8 +540,9 @@ impl IntoShreddingField for (DataType, bool) {
/// should be shredded and with what types. Fields are nullable by default;
pass
/// a `(data_type, nullable)` pair or a `FieldRef` to control nullability.
///
-/// Note: this builder currently only supports struct fields. List support
-/// will be added in the future.
+/// `[*]` represents the shared element schema of a list, so `items[*].id` and
+/// `items[*].name` describe fields on the same list element struct. Numeric
+/// indexes refer to concrete list elements and are rejected by this builder.
///
/// # Example
///
@@ -568,6 +569,8 @@ impl IntoShreddingField for (DataType, bool) {
/// VariantPath::from_iter([VariantPathElement::from("metrics.cpu")]),
/// &DataType::Float64,
/// )?
+/// // [*] describes the shared schema for every element of a list
+/// .with_path("items[*].id", &DataType::Int64)?
/// .build();
/// Ok(())
/// }
@@ -596,6 +599,8 @@ impl ShreddedSchemaBuilder {
/// * `path` - Anything convertible to [`VariantPath`] (e.g., a `&str`)
/// * `field` - Anything convertible via [`IntoShreddingField`] (e.g.
`FieldRef`,
/// `&DataType`, or `(&DataType, bool)` to control nullability)
+ ///
+ /// List schema paths must use `[*]`; numeric indexes return an error.
pub fn with_path<'a, P, F>(mut self, path: P, field: F) -> Result<Self>
where
P: TryInto<VariantPath<'a>>,
@@ -605,7 +610,7 @@ impl ShreddedSchemaBuilder {
let path: VariantPath<'a> = path
.try_into()
.map_err(|e| ArrowError::InvalidArgumentError(format!("{e:?}")))?;
- self.root.insert_path(&path, field.into_shredding_field());
+ self.root.insert_path(&path, field.into_shredding_field())?;
Ok(self)
}
@@ -626,6 +631,8 @@ enum VariantSchemaNode {
Leaf(ShreddingField),
/// An inner struct node with nested fields
Struct(BTreeMap<String, VariantSchemaNode>),
+ /// An inner list node with a shared element schema
+ List(Box<VariantSchemaNode>),
}
impl Default for VariantSchemaNode {
@@ -636,14 +643,18 @@ impl Default for VariantSchemaNode {
impl VariantSchemaNode {
/// Insert a path into this node with the given data type.
- fn insert_path(&mut self, path: &VariantPath<'_>, field: ShreddingField) {
- self.insert_path_elements(path, field);
+ fn insert_path(&mut self, path: &VariantPath<'_>, field: ShreddingField)
-> Result<()> {
+ self.insert_path_elements(path, field)
}
- fn insert_path_elements(&mut self, segments: &[VariantPathElement<'_>],
field: ShreddingField) {
+ fn insert_path_elements(
+ &mut self,
+ segments: &[VariantPathElement<'_>],
+ field: ShreddingField,
+ ) -> Result<()> {
let Some((head, tail)) = segments.split_first() else {
*self = Self::Leaf(field);
- return;
+ return Ok(());
};
match head {
@@ -651,11 +662,11 @@ impl VariantSchemaNode {
// Ensure this node is a Struct node
let children = match self {
Self::Struct(children) => children,
- Self::Leaf(_) => {
+ Self::Leaf(_) | Self::List(_) => {
*self = Self::Struct(BTreeMap::new());
match self {
Self::Struct(children) => children,
- Self::Leaf(_) => unreachable!(),
+ Self::Leaf(_) | Self::List(_) => unreachable!(),
}
}
};
@@ -663,12 +674,25 @@ impl VariantSchemaNode {
children
.entry(name.to_string())
.or_default()
- .insert_path_elements(tail, field);
+ .insert_path_elements(tail, field)
}
- VariantPathElement::Index { .. } => {
- // List support to be added later; reject for now
- unreachable!("List paths are not supported yet");
+ VariantPathElement::ListElement => {
+ let element = match self {
+ Self::List(element) => element,
+ _ => {
+ *self = Self::List(Box::default());
+ match self {
+ Self::List(element) => element,
+ _ => unreachable!(),
+ }
+ }
+ };
+
+ element.insert_path_elements(tail, field)
}
+ VariantPathElement::Index { index } =>
Err(ArrowError::InvalidArgumentError(format!(
+ "List indexes are not supported in schema paths; use [*], got
[{index}]"
+ ))),
}
}
@@ -689,6 +713,7 @@ impl VariantSchemaNode {
Some(DataType::Struct(Fields::from(child_fields)))
}
}
+ Self::List(element) =>
element.to_shredding_field("item").map(DataType::List),
}
}
@@ -699,7 +724,7 @@ impl VariantSchemaNode {
field.data_type.clone(),
field.nullable,
))),
- Self::Struct(_) => self
+ Self::Struct(_) | Self::List(_) => self
.to_shredding_type()
.map(|data_type| Arc::new(Field::new(name, data_type, true))),
}
@@ -1857,15 +1882,12 @@ mod tests {
]);
// Target schema is List<Struct<id:int64,name:utf8>>
- let object_fields = Fields::from(vec![
- Field::new("id", DataType::Int64, true),
- Field::new("name", DataType::Utf8, true),
- ]);
- let list_schema = DataType::List(Arc::new(Field::new(
- "item",
- DataType::Struct(object_fields),
- true,
- )));
+ let list_schema = ShreddedSchemaBuilder::default()
+ .with_path("[*].id", &DataType::Int64)
+ .unwrap()
+ .with_path("[*].name", &DataType::Utf8)
+ .unwrap()
+ .build();
let result = shred_variant(&input, &list_schema).unwrap();
assert_eq!(result.len(), 3);
@@ -2907,6 +2929,67 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_variant_schema_builder_list() -> Result<()> {
+ let shredding_type = ShreddedSchemaBuilder::default()
+ .with_path("items[*].id", &DataType::Int64)?
+ .with_path("items[*].name", &DataType::Utf8)?
+ .build();
+
+ assert_eq!(
+ shredding_type,
+ DataType::Struct(Fields::from(vec![Field::new(
+ "items",
+ DataType::new_list(
+ DataType::Struct(Fields::from(vec![
+ Field::new("id", DataType::Int64, true),
+ Field::new("name", DataType::Utf8, true),
+ ])),
+ true,
+ ),
+ true,
+ )]))
+ );
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_variant_schema_builder_nested_lists() -> Result<()> {
+ let shredding_type = ShreddedSchemaBuilder::default()
+ .with_path("matrix[*][*]", (&DataType::Float64, false))?
+ .build();
+
+ assert_eq!(
+ shredding_type,
+ DataType::Struct(Fields::from(vec![Field::new(
+ "matrix",
+ DataType::new_list(DataType::new_list(DataType::Float64,
false), true),
+ true,
+ )]))
+ );
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_variant_schema_builder_rejects_list_indexes() {
+ for (path, index) in [("items[0].id", 0), ("items[42].name", 42)] {
+ let error = ShreddedSchemaBuilder::default()
+ .with_path(path, &DataType::Int64)
+ .err()
+ .unwrap();
+
+ let ArrowError::InvalidArgumentError(message) = error else {
+ panic!("expected InvalidArgumentError, got {error:?}");
+ };
+ assert_eq!(
+ message,
+ format!("List indexes are not supported in schema paths; use
[*], got [{index}]")
+ );
+ }
+ }
+
#[test]
fn test_variant_schema_builder_with_path_variant_path_arg() -> Result<()> {
let path = VariantPath::from_iter([VariantPathElement::from("a.b")]);
diff --git a/parquet-variant-compute/src/variant_get.rs
b/parquet-variant-compute/src/variant_get.rs
index d8f09f0f1b..681c698b10 100644
--- a/parquet-variant-compute/src/variant_get.rs
+++ b/parquet-variant-compute/src/variant_get.rs
@@ -182,6 +182,9 @@ pub(crate) fn follow_shredded_path_element(
None => Ok(missing_path_step()),
}
}
+ VariantPathElement::ListElement =>
Err(ArrowError::InvalidArgumentError(
+ "variant_get does not support [*] path elements".to_string(),
+ )),
}
}
@@ -455,6 +458,15 @@ pub fn variant_get(input: &ArrayRef, options: GetOptions)
-> Result<ArrayRef> {
cast_options,
} = options;
+ if path
+ .iter()
+ .any(|element| matches!(element, VariantPathElement::ListElement))
+ {
+ return Err(ArrowError::InvalidArgumentError(
+ "variant_get does not support [*] path elements".to_string(),
+ ));
+ }
+
shredded_get_path(&variant_array, &path, as_type.as_deref(), &cast_options)
}
@@ -2474,6 +2486,17 @@ mod test {
);
}
+ #[test]
+ fn test_variant_get_list_element_wildcard_is_invalid_argument() {
+ let (unshredded, _) = create_variant_get_as_variant_test_data();
+ let options =
GetOptions::new_with_path(VariantPath::try_from("field_name[*]").unwrap());
+ let err = variant_get(&unshredded, options).unwrap_err();
+ assert!(
+ matches!(err, ArrowError::InvalidArgumentError(_)),
+ "expected InvalidArgumentError, got {err:?}"
+ );
+ }
+
#[test]
fn test_variant_get_missing_path_as_variant_annotates_value_non_nullable()
{
let (unshredded, shredded) = create_variant_get_as_variant_test_data();
diff --git a/parquet-variant/src/path.rs b/parquet-variant/src/path.rs
index cd41c1cdd9..64aca085c5 100644
--- a/parquet-variant/src/path.rs
+++ b/parquet-variant/src/path.rs
@@ -160,7 +160,7 @@ impl<'a> Deref for VariantPath<'a> {
}
}
-/// Element of a [`VariantPath`] that can be a field name or an index.
+/// Element of a [`VariantPath`] that can be a field name, an index, or a list
element wildcard.
///
/// See [`VariantPath`] for more details and examples.
#[derive(Debug, Clone, PartialEq)]
@@ -169,6 +169,8 @@ pub enum VariantPathElement<'a> {
Field { name: Cow<'a, str> },
/// Access the list element at `index`
Index { index: usize },
+ /// Match the shared element schema of a list (`[*]`)
+ ListElement,
}
impl<'a> VariantPathElement<'a> {
@@ -180,6 +182,10 @@ impl<'a> VariantPathElement<'a> {
pub fn index(index: usize) -> VariantPathElement<'a> {
VariantPathElement::Index { index }
}
+
+ pub fn list_element() -> VariantPathElement<'a> {
+ VariantPathElement::ListElement
+ }
}
// Conversion utilities for `VariantPathElement` from string types
@@ -288,6 +294,15 @@ mod tests {
]);
assert_eq!(path, expected);
+ // list wildcard is distinct from a quoted field named "*"
+ let path = VariantPath::try_from("foo[*]['*']").unwrap();
+ let expected = VariantPath::from_iter([
+ VariantPathElement::field("foo"),
+ VariantPathElement::list_element(),
+ VariantPathElement::field("*"),
+ ]);
+ assert_eq!(path, expected);
+
// invalid index will be treated as field
let path = VariantPath::try_from("foo.bar['abc'][\"def\"]").unwrap();
let expected = VariantPath::from_iter([
@@ -339,13 +354,13 @@ mod tests {
let err = VariantPath::try_from("foo.bar[123abc]").unwrap_err();
assert_eq!(
err.to_string(),
- "Parser error: Invalid token in bracket request: `123abc`.
Expected a quoted string or a number(e.g., `['field']` or `[123]`)"
+ "Parser error: Invalid token in bracket request: `123abc`.
Expected `*`, a quoted string, or a number(e.g., `[*]`, `['field']`, or
`[123]`)"
);
let err = VariantPath::try_from("foo.bar[abc]").unwrap_err();
assert_eq!(
err.to_string(),
- "Parser error: Invalid token in bracket request: `abc`. Expected a
quoted string or a number(e.g., `['field']` or `[123]`)"
+ "Parser error: Invalid token in bracket request: `abc`. Expected
`*`, a quoted string, or a number(e.g., `[*]`, `['field']`, or `[123]`)"
);
// Out-of-range integer indexes are invalid path tokens.
diff --git a/parquet-variant/src/utils.rs b/parquet-variant/src/utils.rs
index 10f0a5a721..782f718cfc 100644
--- a/parquet-variant/src/utils.rs
+++ b/parquet-variant/src/utils.rs
@@ -267,10 +267,12 @@ fn parse_in_bracket(s: &str, i: usize) ->
Result<(VariantPathElement<'_>, usize)
{
// Quoted field name, e.g., ['field'] or ['123'] or ["123"]
VariantPathElement::field(inner.to_string())
+ } else if unescaped == "*" {
+ VariantPathElement::list_element()
} else {
let Ok(idx) = unescaped.parse() else {
return Err(ArrowError::ParseError(format!(
- "Invalid token in bracket request: `{unescaped}`. Expected a
quoted string or a number(e.g., `['field']` or `[123]`)"
+ "Invalid token in bracket request: `{unescaped}`. Expected
`*`, a quoted string, or a number(e.g., `[*]`, `['field']`, or `[123]`)"
)));
};
VariantPathElement::index(idx)
diff --git a/parquet-variant/src/variant.rs b/parquet-variant/src/variant.rs
index dcb81de490..6e57e0bee5 100644
--- a/parquet-variant/src/variant.rs
+++ b/parquet-variant/src/variant.rs
@@ -1568,6 +1568,7 @@ impl<'m, 'v> Variant<'m, 'v> {
.try_fold(self.clone(), |output, element| match element {
VariantPathElement::Field { name } =>
output.get_object_field(name),
VariantPathElement::Index { index } =>
output.get_list_element(*index),
+ VariantPathElement::ListElement => None,
})
}
}
