This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a commit to branch branch-1.11
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/branch-1.11 by this push:
new 0bb285d1a AVRO-3709: [Rust] Add aliases to record fields (#2087)
0bb285d1a is described below
commit 0bb285d1adac861ce88770a168ac7548d086f17a
Author: Martin Grigorov <[email protected]>
AuthorDate: Thu Feb 9 09:23:09 2023 +0200
AVRO-3709: [Rust] Add aliases to record fields (#2087)
* AVRO-3709: [Rust] Add 'aliases' field to RecordField
* AVRO-3709: [Rust] Add support for serializing RecordField's aliases
* AVRO-3709: [Rust] Record field's aliases don't have namespace
Use String instead of Alias.
* AVRO-3709: [Rust] Add support for field aliases in avro_derive
---------
Signed-off-by: Martin Tzvetanov Grigorov <[email protected]>
(cherry picked from commit a512fa2fdf1b370548446e83ac48b9db52e04275)
---
lang/rust/avro/src/schema.rs | 76 ++++++++++++++++++++++++++++++++++++++--
lang/rust/avro/src/types.rs | 19 ++++++----
lang/rust/avro/tests/schema.rs | 1 +
lang/rust/avro_derive/src/lib.rs | 31 ++++++++++++++--
4 files changed, 116 insertions(+), 11 deletions(-)
diff --git a/lang/rust/avro/src/schema.rs b/lang/rust/avro/src/schema.rs
index 02e7c37be..0b21ad659 100644
--- a/lang/rust/avro/src/schema.rs
+++ b/lang/rust/avro/src/schema.rs
@@ -563,6 +563,8 @@ pub struct RecordField {
pub name: String,
/// Documentation of the field.
pub doc: Documentation,
+ /// Aliases of the field's name. They have no namespace.
+ pub aliases: Option<Vec<String>>,
/// Default value of the field.
/// This value will be used when reading Avro datum if schema resolution
/// is enabled.
@@ -603,6 +605,16 @@ impl RecordField {
let default = field.get("default").cloned();
+ let aliases = field.get("aliases").and_then(|aliases| {
+ aliases.as_array().map(|aliases| {
+ aliases
+ .iter()
+ .flat_map(|alias| alias.as_str())
+ .map(|alias| alias.to_string())
+ .collect::<Vec<String>>()
+ })
+ });
+
let order = field
.get("order")
.and_then(|order| order.as_str())
@@ -613,6 +625,7 @@ impl RecordField {
name,
doc: field.doc(),
default,
+ aliases,
schema,
order,
position,
@@ -1268,6 +1281,12 @@ impl Parser {
for field in &fields {
lookup.insert(field.name.clone(), field.position);
+
+ if let Some(ref field_aliases) = field.aliases {
+ for alias in field_aliases {
+ lookup.insert(alias.clone(), field.position);
+ }
+ }
}
let schema = Schema::Record {
@@ -1675,6 +1694,10 @@ impl Serialize for RecordField {
map.serialize_entry("default", default)?;
}
+ if let Some(ref aliases) = self.aliases {
+ map.serialize_entry("aliases", aliases)?;
+ }
+
map.end()
}
}
@@ -2067,6 +2090,7 @@ mod tests {
name: "next".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::Union(
UnionSchema::new(vec![
Schema::Null,
@@ -2090,6 +2114,7 @@ mod tests {
name: "next".to_string(),
doc: None,
default: Some(json!(2)),
+ aliases: None,
schema: Schema::Long,
order: RecordFieldOrder::Ascending,
position: 1,
@@ -2144,6 +2169,7 @@ mod tests {
name: "field_one".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::Union(
UnionSchema::new(vec![
Schema::Ref {
@@ -2235,6 +2261,7 @@ mod tests {
name: "field_one".to_string(),
doc: None,
default: Some(Value::Null),
+ aliases: None,
schema: Schema::Union(
UnionSchema::new(vec![
Schema::Null,
@@ -2284,6 +2311,7 @@ mod tests {
name: "a".to_string(),
doc: None,
default: Some(Value::Number(42i64.into())),
+ aliases: None,
schema: Schema::Long,
order: RecordFieldOrder::Ascending,
position: 0,
@@ -2293,6 +2321,7 @@ mod tests {
name: "b".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::String,
order: RecordFieldOrder::Ascending,
position: 1,
@@ -2345,6 +2374,7 @@ mod tests {
name: "recordField".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::Record {
name: Name::new("Node").unwrap(),
aliases: None,
@@ -2354,6 +2384,7 @@ mod tests {
name: "label".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::String,
order: RecordFieldOrder::Ascending,
position: 0,
@@ -2363,6 +2394,7 @@ mod tests {
name: "children".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::Array(Box::new(Schema::Ref {
name: Name::new("Node").unwrap(),
})),
@@ -2522,6 +2554,7 @@ mod tests {
name: "value".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::Long,
order: RecordFieldOrder::Ascending,
position: 0,
@@ -2531,6 +2564,7 @@ mod tests {
name: "next".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::Union(
UnionSchema::new(vec![
Schema::Null,
@@ -2591,6 +2625,7 @@ mod tests {
name: "value".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::Long,
order: RecordFieldOrder::Ascending,
position: 0,
@@ -2600,6 +2635,7 @@ mod tests {
name: "next".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::Ref {
name: Name {
name: "record".to_owned(),
@@ -2658,6 +2694,7 @@ mod tests {
name: "enum".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::Enum {
name: Name {
name: "enum".to_owned(),
@@ -2676,6 +2713,7 @@ mod tests {
name: "next".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::Enum {
name: Name {
name: "enum".to_owned(),
@@ -2738,6 +2776,7 @@ mod tests {
name: "fixed".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::Fixed {
name: Name {
name: "fixed".to_owned(),
@@ -2756,6 +2795,7 @@ mod tests {
name: "next".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::Fixed {
name: Name {
name: "fixed".to_owned(),
@@ -3869,7 +3909,13 @@ mod tests {
"namespace": "space",
"aliases": ["b", "x.y", ".c"],
"fields" : [
- {"name": "time", "type": "long"}
+ {
+ "name": "time",
+ "type": "long",
+ "doc": "The documentation is not serialized",
+ "default": 123,
+ "aliases": ["time1", "ns.time2"]
+ }
]
}
"#,
@@ -3879,7 +3925,7 @@ mod tests {
let value = serde_json::to_value(&schema).unwrap();
let serialized = serde_json::to_string(&value).unwrap();
assert_eq!(
-
r#"{"aliases":["space.b","x.y","c"],"fields":[{"name":"time","type":"long"}],"name":"a","namespace":"space","type":"record"}"#,
+
r#"{"aliases":["space.b","x.y","c"],"fields":[{"aliases":["time1","ns.time2"],"default":123,"name":"time","type":"long"}],"name":"a","namespace":"space","type":"record"}"#,
&serialized
);
assert_eq!(schema, Schema::parse_str(&serialized).unwrap());
@@ -4237,4 +4283,30 @@ mod tests {
_ => panic!("Expected Schema::Record"),
}
}
+
+ #[test]
+ fn avro_3709_parsing_of_record_field_aliases() {
+ let schema = r#"
+ {
+ "name": "rec",
+ "type": "record",
+ "fields": [
+ {
+ "name": "num",
+ "type": "int",
+ "aliases": ["num1", "num2"]
+ }
+ ]
+ }
+ "#;
+
+ let schema = Schema::parse_str(schema).unwrap();
+ if let Schema::Record { fields, .. } = schema {
+ let num_field = &fields[0];
+ assert_eq!(num_field.name, "num");
+ assert_eq!(num_field.aliases, Some(vec!("num1".into(),
"num2".into())));
+ } else {
+ panic!("Expected a record schema!");
+ }
+ }
}
diff --git a/lang/rust/avro/src/types.rs b/lang/rust/avro/src/types.rs
index 9f3f3d253..60f94ecf8 100644
--- a/lang/rust/avro/src/types.rs
+++ b/lang/rust/avro/src/types.rs
@@ -1107,6 +1107,7 @@ mod tests {
name: "field_name".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::Int,
order: RecordFieldOrder::Ignore,
position: 0,
@@ -1116,7 +1117,7 @@ mod tests {
attributes: Default::default(),
},
false,
- r#"Invalid value: Record([("unknown_field_name", Null)]) for
schema: Record { name: Name { name: "record_name", namespace: None }, aliases:
None, doc: None, fields: [RecordField { name: "field_name", doc: None, default:
None, schema: Int, order: Ignore, position: 0, custom_attributes: {} }],
lookup: {}, attributes: {} }. Reason: There is no schema field for field
'unknown_field_name'"#,
+ r#"Invalid value: Record([("unknown_field_name", Null)]) for
schema: Record { name: Name { name: "record_name", namespace: None }, aliases:
None, doc: None, fields: [RecordField { name: "field_name", doc: None, aliases:
None, default: None, schema: Int, order: Ignore, position: 0,
custom_attributes: {} }], lookup: {}, attributes: {} }. Reason: There is no
schema field for field 'unknown_field_name'"#,
),
(
Value::Record(vec![("field_name".to_string(), Value::Null)]),
@@ -1128,6 +1129,7 @@ mod tests {
name: "field_name".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::Ref {
name: Name::new("missing").unwrap(),
},
@@ -1139,7 +1141,7 @@ mod tests {
attributes: Default::default(),
},
false,
- r#"Invalid value: Record([("field_name", Null)]) for schema:
Record { name: Name { name: "record_name", namespace: None }, aliases: None,
doc: None, fields: [RecordField { name: "field_name", doc: None, default: None,
schema: Ref { name: Name { name: "missing", namespace: None } }, order: Ignore,
position: 0, custom_attributes: {} }], lookup: {"field_name": 0}, attributes:
{} }. Reason: Unresolved schema reference: 'Name { name: "missing", namespace:
None }'. Parsed names: []"#,
+ r#"Invalid value: Record([("field_name", Null)]) for schema:
Record { name: Name { name: "record_name", namespace: None }, aliases: None,
doc: None, fields: [RecordField { name: "field_name", doc: None, aliases: None,
default: None, schema: Ref { name: Name { name: "missing", namespace: None } },
order: Ignore, position: 0, custom_attributes: {} }], lookup: {"field_name":
0}, attributes: {} }. Reason: Unresolved schema reference: 'Name { name:
"missing", namespace: None } [...]
),
];
@@ -1287,6 +1289,7 @@ mod tests {
name: "a".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::Long,
order: RecordFieldOrder::Ascending,
position: 0,
@@ -1296,6 +1299,7 @@ mod tests {
name: "b".to_string(),
doc: None,
default: None,
+ aliases: None,
schema: Schema::String,
order: RecordFieldOrder::Ascending,
position: 1,
@@ -1305,6 +1309,7 @@ mod tests {
name: "c".to_string(),
doc: None,
default: Some(JsonValue::Null),
+ aliases: None,
schema: Schema::Union(
UnionSchema::new(vec![Schema::Null,
Schema::Int]).unwrap(),
),
@@ -1342,7 +1347,7 @@ mod tests {
]);
assert!(!value.validate(&schema));
assert_logged(
- r#"Invalid value: Record([("a", Boolean(false)), ("b",
String("foo"))]) for schema: Record { name: Name { name: "some_record",
namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a",
doc: None, default: None, schema: Long, order: Ascending, position: 0,
custom_attributes: {} }, RecordField { name: "b", doc: None, default: None,
schema: String, order: Ascending, position: 1, custom_attributes: {} },
RecordField { name: "c", doc: None, default: Some(Null) [...]
+ r#"Invalid value: Record([("a", Boolean(false)), ("b",
String("foo"))]) for schema: Record { name: Name { name: "some_record",
namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a",
doc: None, aliases: None, default: None, schema: Long, order: Ascending,
position: 0, custom_attributes: {} }, RecordField { name: "b", doc: None,
aliases: None, default: None, schema: String, order: Ascending, position: 1,
custom_attributes: {} }, RecordField { name: "c", [...]
);
let value = Value::Record(vec![
@@ -1351,7 +1356,7 @@ mod tests {
]);
assert!(!value.validate(&schema));
assert_logged(
- r#"Invalid value: Record([("a", Long(42)), ("c", String("foo"))])
for schema: Record { name: Name { name: "some_record", namespace: None },
aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, default:
None, schema: Long, order: Ascending, position: 0, custom_attributes: {} },
RecordField { name: "b", doc: None, default: None, schema: String, order:
Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc:
None, default: Some(Null), sche [...]
+ r#"Invalid value: Record([("a", Long(42)), ("c", String("foo"))])
for schema: Record { name: Name { name: "some_record", namespace: None },
aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases:
None, default: None, schema: Long, order: Ascending, position: 0,
custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None,
default: None, schema: String, order: Ascending, position: 1,
custom_attributes: {} }, RecordField { name: "c", doc: N [...]
);
assert_not_logged(
r#"Invalid value: String("foo") for schema: Int. Reason:
Unsupported value-schema combination"#,
@@ -1363,7 +1368,7 @@ mod tests {
]);
assert!(!value.validate(&schema));
assert_logged(
- r#"Invalid value: Record([("a", Long(42)), ("d", String("foo"))])
for schema: Record { name: Name { name: "some_record", namespace: None },
aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, default:
None, schema: Long, order: Ascending, position: 0, custom_attributes: {} },
RecordField { name: "b", doc: None, default: None, schema: String, order:
Ascending, position: 1, custom_attributes: {} }, RecordField { name: "c", doc:
None, default: Some(Null), sche [...]
+ r#"Invalid value: Record([("a", Long(42)), ("d", String("foo"))])
for schema: Record { name: Name { name: "some_record", namespace: None },
aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, aliases:
None, default: None, schema: Long, order: Ascending, position: 0,
custom_attributes: {} }, RecordField { name: "b", doc: None, aliases: None,
default: None, schema: String, order: Ascending, position: 1,
custom_attributes: {} }, RecordField { name: "c", doc: N [...]
);
let value = Value::Record(vec![
@@ -1374,7 +1379,7 @@ mod tests {
]);
assert!(!value.validate(&schema));
assert_logged(
- r#"Invalid value: Record([("a", Long(42)), ("b", String("foo")),
("c", Null), ("d", Null)]) for schema: Record { name: Name { name:
"some_record", namespace: None }, aliases: None, doc: None, fields:
[RecordField { name: "a", doc: None, default: None, schema: Long, order:
Ascending, position: 0, custom_attributes: {} }, RecordField { name: "b", doc:
None, default: None, schema: String, order: Ascending, position: 1,
custom_attributes: {} }, RecordField { name: "c", doc: None, [...]
+ r#"Invalid value: Record([("a", Long(42)), ("b", String("foo")),
("c", Null), ("d", Null)]) for schema: Record { name: Name { name:
"some_record", namespace: None }, aliases: None, doc: None, fields:
[RecordField { name: "a", doc: None, aliases: None, default: None, schema:
Long, order: Ascending, position: 0, custom_attributes: {} }, RecordField {
name: "b", doc: None, aliases: None, default: None, schema: String, order:
Ascending, position: 1, custom_attributes: {} }, Recor [...]
);
assert!(Value::Map(
@@ -1394,7 +1399,7 @@ mod tests {
)
.validate(&schema));
assert_logged(
- r#"Invalid value: Map({"d": Long(123)}) for schema: Record { name:
Name { name: "some_record", namespace: None }, aliases: None, doc: None,
fields: [RecordField { name: "a", doc: None, default: None, schema: Long,
order: Ascending, position: 0, custom_attributes: {} }, RecordField { name:
"b", doc: None, default: None, schema: String, order: Ascending, position: 1,
custom_attributes: {} }, RecordField { name: "c", doc: None, default:
Some(Null), schema: Union(UnionSchema { sc [...]
+ r#"Invalid value: Map({"d": Long(123)}) for schema: Record { name:
Name { name: "some_record", namespace: None }, aliases: None, doc: None,
fields: [RecordField { name: "a", doc: None, aliases: None, default: None,
schema: Long, order: Ascending, position: 0, custom_attributes: {} },
RecordField { name: "b", doc: None, aliases: None, default: None, schema:
String, order: Ascending, position: 1, custom_attributes: {} }, RecordField {
name: "c", doc: None, aliases: None, defaul [...]
Field with name '"b"' is not a member of the map items"#,
);
diff --git a/lang/rust/avro/tests/schema.rs b/lang/rust/avro/tests/schema.rs
index 4f7270d82..0789eb9b4 100644
--- a/lang/rust/avro/tests/schema.rs
+++ b/lang/rust/avro/tests/schema.rs
@@ -896,6 +896,7 @@ fn test_parse_reused_record_schema_by_fullname() {
ref name,
doc: _,
default: _,
+ aliases: _,
ref schema,
order: _,
position: _,
diff --git a/lang/rust/avro_derive/src/lib.rs b/lang/rust/avro_derive/src/lib.rs
index 77b0c6a24..369bcfdb6 100644
--- a/lang/rust/avro_derive/src/lib.rs
+++ b/lang/rust/avro_derive/src/lib.rs
@@ -30,6 +30,8 @@ struct FieldOptions {
doc: Option<String>,
#[darling(default)]
default: Option<String>,
+ #[darling(multiple)]
+ alias: Vec<String>,
#[darling(default)]
rename: Option<String>,
#[darling(default)]
@@ -148,6 +150,7 @@ fn get_data_struct_schema_def(
}
None => quote! { None },
};
+ let aliases = preserve_vec(field_attrs.alias);
let schema_expr = type_to_schema_expr(&field.ty)?;
let position = index;
record_field_exprs.push(quote! {
@@ -155,6 +158,7 @@ fn get_data_struct_schema_def(
name: #name.to_string(),
doc: #doc,
default: #default_value,
+ aliases: #aliases,
schema: #schema_expr,
order:
apache_avro::schema::RecordFieldOrder::Ascending,
position: #position,
@@ -455,8 +459,9 @@ mod tests {
match syn::parse2::<DeriveInput>(test_struct) {
Ok(mut input) => {
- assert!(derive_avro_schema(&mut input).is_ok());
- assert!(derive_avro_schema(&mut input)
+ let schema_token_stream = derive_avro_schema(&mut input);
+ assert!(&schema_token_stream.is_ok());
+ assert!(schema_token_stream
.unwrap()
.to_string()
.contains("namespace.testing"))
@@ -492,4 +497,26 @@ mod tests {
assert_eq!(type_path_schema_expr(&syn::parse2::<TypePath>(quote!{Vec<T>}).unwrap()).to_string(),
quote!{<Vec<T> as
apache_avro::schema::derive::AvroSchemaComponent>::get_schema_in_ctxt(named_schemas,
enclosing_namespace)}.to_string());
assert_eq!(type_path_schema_expr(&syn::parse2::<TypePath>(quote!{AnyType}).unwrap()).to_string(),
quote!{<AnyType as
apache_avro::schema::derive::AvroSchemaComponent>::get_schema_in_ctxt(named_schemas,
enclosing_namespace)}.to_string());
}
+
+ #[test]
+ fn test_avro_3709_record_field_attributes() {
+ let test_struct = quote! {
+ struct A {
+ #[avro(alias = "a1", alias = "a2", doc = "a doc", default =
"123", rename = "a3")]
+ a: i32
+ }
+ };
+
+ match syn::parse2::<DeriveInput>(test_struct) {
+ Ok(mut input) => {
+ let schema_res = derive_avro_schema(&mut input);
+ let expected_token_stream = r#"let schema_fields = vec !
[apache_avro :: schema :: RecordField { name : "a3" . to_string () , doc : Some
("a doc" . into ()) , default : Some (serde_json :: from_str ("123") . expect
(format ! ("Invalid JSON: {:?}" , "123") . as_str ())) , aliases : Some (vec !
["a1" . into () , "a2" . into ()]) , schema : apache_avro :: schema :: Schema
:: Int , order : apache_avro :: schema :: RecordFieldOrder :: Ascending ,
position : 0usize , custom_att [...]
+ let schema_token_stream = schema_res.unwrap().to_string();
+ assert!(schema_token_stream.contains(expected_token_stream));
+ }
+ Err(error) => panic!(
+ "Failed to parse as derive input when it should be able to.
Error: {error:?}"
+ ),
+ };
+ }
}
