This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 0b94195639 fix(arrow-avro): resolve named types declared with a full
name (#11730)
0b94195639 is described below
commit 0b94195639ccbae0d615596cdb11445cde1441b6
Author: Sebastian Turner <[email protected]>
AuthorDate: Tue Oct 27 08:54:40 2026 +0000
fix(arrow-avro): resolve named types declared with a full name (#11730)
# Which issue does this PR close?
- Closes #NNN
# Rationale for this change
Per the Avro spec, a name containing a dot is a full name. Its namespace
attribute is ignored and nested types inherit the namespace taken from
the name. The named type resolver in `codec.rs` did not apply this rule
when registering types, so a type declared as `ns.Status` could not be
referenced as `ns.Status`, and a type nested in a record named
`com.example.Event` could not be referenced as `com.example.Kind`. Both
fail with `Failed to resolve ...`.
# What changes are included in this PR?
- `Resolver::register` splits a dotted name into namespace and name, the
same way `Resolver::resolve` already does.
- A small `named_type_namespace` helper returns the namespace of a named
type, taking it from the name when the name is dotted. It replaces the
`x.namespace.or(namespace)` expressions for records, enums and fixed
types in both `parse_type` and the schema resolution paths, so nested
types inherit the right namespace.
# Are these changes tested?
Yes. `test_named_type_reuse_with_full_name` covers both a dotted enum
name referenced by its full name and a nested record referenced through
the namespace of a dotted enclosing record name. It fails on `main` with
`Failed to resolve enum_ns.Status`.
# Are there any significant user-facing changes?
No API changes. Schemas that use dotted names now parse instead of
returning an error. For a named type declared with a dotted name, the
`avro.namespace` field metadata now holds the namespace from that name
rather than the enclosing one.
Fixes #11729
AI tools were used to find this bug and to write the fix, the test and
this description. The tests were run locally before and after the
change.
Signed-off-by: Sebastian Turner <[email protected]>
Co-authored-by: Jeffrey Vo <[email protected]>
---
arrow-avro/src/codec.rs | 88 +++++++++++++++++++++++++++++++++++++++++++++----
1 file changed, 81 insertions(+), 7 deletions(-)
diff --git a/arrow-avro/src/codec.rs b/arrow-avro/src/codec.rs
index 34d5ea9437..97e82cd42c 100644
--- a/arrow-avro/src/codec.rs
+++ b/arrow-avro/src/codec.rs
@@ -1236,7 +1236,10 @@ struct Resolver<'a> {
impl<'a> Resolver<'a> {
fn register(&mut self, name: &'a str, namespace: Option<&'a str>, schema:
AvroDataType) {
- self.map.insert((namespace.unwrap_or(""), name), schema);
+ let (namespace, name) = name
+ .rsplit_once('.')
+ .unwrap_or_else(|| (namespace.unwrap_or(""), name));
+ self.map.insert((namespace, name), schema);
}
fn resolve(&self, name: &str, namespace: Option<&'a str>) ->
Result<AvroDataType, ArrowError> {
@@ -1250,6 +1253,23 @@ impl<'a> Resolver<'a> {
}
}
+/// Returns the namespace of a named type.
+///
+/// A name containing a dot is a full name, so its namespace is taken from the
name and both
+/// the `namespace` attribute and the enclosing namespace are ignored.
+///
+/// See <https://avro.apache.org/docs/1.11.1/specification/#names>
+fn named_type_namespace<'a>(
+ name: &'a str,
+ namespace: Option<&'a str>,
+ enclosing_ns: Option<&'a str>,
+) -> Option<&'a str> {
+ match name.rsplit_once('.') {
+ Some((ns, _)) => Some(ns),
+ None => namespace.or(enclosing_ns),
+ }
+}
+
fn full_name_set(name: &str, ns: Option<&str>, aliases: &[&str]) ->
HashSet<String> {
let mut out = HashSet::with_capacity(1 + aliases.len());
let (full, _) = make_full_name(name, ns, None);
@@ -1502,7 +1522,7 @@ impl<'a> Maker<'a> {
}
Schema::Complex(c) => match c {
ComplexType::Record(r) => {
- let namespace = r.namespace.or(namespace);
+ let namespace = named_type_namespace(r.name, r.namespace,
namespace);
let mut metadata = r.attributes.field_metadata();
let fields = r
.fields
@@ -1540,7 +1560,7 @@ impl<'a> Maker<'a> {
let size = f.size.try_into().map_err(|e| {
ArrowError::ParseError(format!("Overflow converting
size to i32: {e}"))
})?;
- let namespace = f.namespace.or(namespace);
+ let namespace = named_type_namespace(f.name, f.namespace,
namespace);
let mut metadata = f.attributes.field_metadata();
metadata.insert(AVRO_NAME_METADATA_KEY.to_string(),
f.name.to_string());
if let Some(ns) = namespace {
@@ -1630,7 +1650,7 @@ impl<'a> Maker<'a> {
Ok(field)
}
ComplexType::Enum(e) => {
- let namespace = e.namespace.or(namespace);
+ let namespace = named_type_namespace(e.name, e.namespace,
namespace);
let symbols = e
.symbols
.iter()
@@ -2224,7 +2244,7 @@ impl<'a> Maker<'a> {
mapping: Arc::from(mapping),
default_index,
}));
- let reader_ns = reader_enum.namespace.or(namespace);
+ let reader_ns = named_type_namespace(reader_enum.name,
reader_enum.namespace, namespace);
self.resolver
.register(reader_enum.name, reader_ns, dt.clone());
Ok(dt)
@@ -2272,8 +2292,10 @@ impl<'a> Maker<'a> {
reader_record.namespace,
&reader_record.aliases,
)?;
- let writer_ns = writer_record.namespace.or(namespace);
- let reader_ns = reader_record.namespace.or(namespace);
+ let writer_ns =
+ named_type_namespace(writer_record.name, writer_record.namespace,
namespace);
+ let reader_ns =
+ named_type_namespace(reader_record.name, reader_record.namespace,
namespace);
let mut reader_md = reader_record.attributes.field_metadata();
reader_md.insert(
AVRO_NAME_METADATA_KEY.to_string(),
@@ -3284,6 +3306,58 @@ mod tests {
}
}
+ #[test]
+ fn test_named_type_reuse_with_full_name() {
+ // A name containing a dot is a full name, and types nested in a record
+ // with a full name inherit the namespace from that name.
+ let schema_str = r#"
+ {
+ "type": "record",
+ "name": "record_ns.Record",
+ "fields": [
+ {
+ "name": "status",
+ "type": {
+ "type": "enum",
+ "name": "enum_ns.Status",
+ "symbols": ["ACTIVE", "INACTIVE"]
+ }
+ },
+ { "name": "backupStatus", "type": "enum_ns.Status" },
+ {
+ "name": "nested",
+ "type": {
+ "type": "record",
+ "name": "Nested",
+ "fields": [{ "name": "nested_int", "type": "int" }]
+ }
+ },
+ { "name": "nestedAgain", "type": "record_ns.Nested" }
+ ]
+ }
+ "#;
+
+ let schema: Schema = serde_json::from_str(schema_str).unwrap();
+
+ let mut maker = Maker::new(false, false, Tz::default());
+ let avro_data_type = maker.make_data_type(&schema, None,
None).unwrap();
+
+ let Codec::Struct(fields) = avro_data_type.codec() else {
+ panic!("Top-level schema is not a struct");
+ };
+ assert_eq!(fields.len(), 4);
+ assert!(matches!(fields[1].data_type().codec(), Codec::Enum(_)));
+ assert_eq!(
+ fields[1].data_type().codec().data_type(),
+ fields[0].data_type().codec().data_type()
+ );
+ assert!(matches!(fields[3].data_type().codec(), Codec::Struct(_)));
+ assert_eq!(
+ fields[3].data_type().codec().data_type(),
+ fields[2].data_type().codec().data_type()
+ );
+ }
+
#[test]
fn
test_resolve_from_writer_and_reader_defaults_root_name_for_non_record_reader() {
let writer_schema =
Schema::TypeName(TypeName::Primitive(PrimitiveType::String));
